File
Blob: src/workerd/io/worker-fs.h
| 1 | #pragma once |
| 2 | |
| 3 | #include <workerd/jsg/jsg.h> |
| 4 | #include <workerd/jsg/url.h> |
| 5 | |
| 6 | #include <kj/common.h> |
| 7 | #include <kj/refcount.h> |
| 8 | #include <kj/time.h> |
| 9 | |
| 10 | // Every worker instance will have its own root directory (/). In this root |
| 11 | // directory we will have at least three special directories, the "bundle" root, |
| 12 | // the "dev" root, and the "temp" root. More special directories can be added |
| 13 | // later. |
| 14 | // |
| 15 | // The bundle root is where all of the modules that are included in the worker |
| 16 | // bundle will be accessible. Everything in this directory will be strictly |
| 17 | // read-only. The contents are populated by the worker configuration bundle. |
| 18 | // By default, the bundle root will be /bundle but this can be overridden |
| 19 | // using the FsMap API defined below. |
| 20 | // |
| 21 | // The dev root is where special "device" files will be accessible. For example, |
| 22 | // the /dev/null, /dev/zero, /dev/full, and /dev/random files will be here. |
| 23 | // By default these are in /dev but the location can be overridden using the |
| 24 | // FsMap API. |
| 25 | // |
| 26 | // The temp root is where we will allow temporary files to be created. |
| 27 | // Everything in this directory is read-write but will be transient. By default, |
| 28 | // the temp root will be /tmp but this can also be overridden. |
| 29 | // |
| 30 | // Let's imagine the following simple workerd configuration: |
| 31 | // |
| 32 | // ``` |
| 33 | // const helloWorld :Workerd.Worker = ( |
| 34 | // modules = [ |
| 35 | // (name = "worker", |
| 36 | // esModule = embed "worker.js"), |
| 37 | // (name = "foo", text = "Hello World!"), |
| 38 | // ], |
| 39 | // compatibilityDate = "2023-02-28", |
| 40 | // ); |
| 41 | // ``` |
| 42 | // |
| 43 | // Given this configuration, the worker fs will initially have the following |
| 44 | // structure: |
| 45 | // |
| 46 | // / |
| 47 | // ├── bundle |
| 48 | // │ ├── worker |
| 49 | // │ └── foo |
| 50 | // ├── dev |
| 51 | // │ ├── null |
| 52 | // │ ├── zero |
| 53 | // │ ├── full |
| 54 | // │ └── random |
| 55 | // └── tmp |
| 56 | // |
| 57 | // We can access the filesystem using VirtualFileSystem::current(): |
| 58 | // |
| 59 | // ```cpp |
| 60 | // jsg::Lock& js = ... |
| 61 | // auto& vfs = VirtualFileSystem::current(js); |
| 62 | // // Resolve the root of the file system |
| 63 | // KJ_IF_SOME(node, vfs.resolve(js, "file:///path/to/thing"_url)) { |
| 64 | // KJ_SWITCH_ONEOF(node) { |
| 65 | // KJ_CASE_ONEOF(file, kj::Rc<File>) { |
| 66 | // // ... |
| 67 | // } |
| 68 | // KJ_CASE_ONEOF(dir, kj::Rc<Directory>) { |
| 69 | // // ... |
| 70 | // } |
| 71 | // KJ_CASE_ONEOF(link, kj::Rc<SymbolicLink>) { |
| 72 | // // ... |
| 73 | // } |
| 74 | // } |
| 75 | // } |
| 76 | // ``` |
| 77 | // |
| 78 | // The temporary file directory is a bit special in that the contents are fully |
| 79 | // transient based on whether there is an active IoContext or not. We use a |
| 80 | // special RAII TmpDirStoreScope scope to manage the contents of the temporary |
| 81 | // directory. For example, |
| 82 | // |
| 83 | // ```cpp |
| 84 | // KJ_IF_SOME(node, vfs.resolve("file:///tmp"_url)) { |
| 85 | // auto& dir = KJ_ASSERT_NONNULL(node.tryGet<kj::Rc<Directory>>()); |
| 86 | // TmpDirStoreScope temp_dir_scope; |
| 87 | // kj::Path path("a/b/c/foo.txt") |
| 88 | // auto tmpFile = dir.tryOpen(js, path, { FsType::FILE }); |
| 89 | // KJ_ASSERT(tmpFile.write(js, 0, "Hello World!"_kjb) == 12); |
| 90 | // // The temp dir scope is destructed and the file is deleted |
| 91 | // } |
| 92 | // ``` |
| 93 | // |
| 94 | // If there is an active IoContext, the temporary file will instead be created |
| 95 | // within that IoContext's TmpDirStoreScope, and will be deleted when the |
| 96 | // IoContext is destructed. This allows us to have a single virtual file |
| 97 | // system whose temporary directories are either deleted immediately as soon |
| 98 | // as the execution scope is exited, or are specific to the IoContext and are |
| 99 | // deleted when the IoContext is destructed. This mechanism allows us to have |
| 100 | // multiple IoContexts active at the same time while still having a single |
| 101 | // virtual file system whose contents correctly reflect the current IoContext. |
| 102 | // |
| 103 | // Note that operations on files and directories require having a jsg::Lock&. |
| 104 | // This is used for several purposes. One, for writable files it is used to |
| 105 | // provide access to the memory accounting systen, ensuring that the memory |
| 106 | // held by writable files is appropriately accounted for towards to isolate |
| 107 | // heap limit. Second, use of the jsg::Lock& ensures that file system mutations |
| 108 | // are performed in a thread-safe manner -- specifically, we use it to ensure |
| 109 | // that only one thread is allowed to access the file system at a time since |
| 110 | // only one thread at a time can hold the jsg::Lock. |
| 111 | // |
| 112 | // The design here is intended to be extensible. We can add new root directories |
| 113 | // in the future with different semantics and implementations. For example, to |
| 114 | // support python workers, for instance, we can introduce a new root directory |
| 115 | // that is backed by a tar/zip file containing the python standard library, etc. |
| 116 | // |
| 117 | // To support the implementation of node:fs the virtual file system needs a |
| 118 | // concept of file descriptors that map somewhat cleanly to the posix notion |
| 119 | // of file descriptors. This is a bit tricky because fds are a finite resource. |
| 120 | // The VFS implementation uses a notion of file descriptors that are scoped |
| 121 | // specifically to the worker... that is, fd 1 in one worker is not the same as |
| 122 | // fd 1 in another. It will be a requirement that fd's are closed explicitly or |
| 123 | // they may still "leak" beyond the scope where they are used, but when the |
| 124 | // worker is torn down, all associated file descriptors will be automatically |
| 125 | // destroyed/closed rather than leaking for the entire process. We use just a |
| 126 | // simple in-memory table to track the file descriptors and their associated |
| 127 | // files/directories. |
| 128 | // |
| 129 | // Note: It is important to keep in mind that Directory, File, and SymbolicLink |
| 130 | // instances are kj::Refcounted objects. We utilize the Isolate lock to safely |
| 131 | // manage the refcounts to avoid having to make these AtomicRefcounted, which |
| 132 | // would carry additional overhead. This means that it is essentially that all |
| 133 | // references to these objects, and all operations on them, as well as dropping |
| 134 | // the references, are done while holding the isolate lock. It is particularly |
| 135 | // necessary to take care when capturing these objects, for instance, into a |
| 136 | // kj Promise then lambda. If the promise is dropped outside of the isolate lock, |
| 137 | // the refcount will be decremented outside of the lock, causing issues. The |
| 138 | // bottom line is that you should always be holding the isolate lock when |
| 139 | // interacting with the virtual file system. |
| 140 | namespace workerd { |
| 141 | |
| 142 | // TODO(node-fs): Currently, all files and directories use a fixed last |
| 143 | // modified time set to the Unix epoch. This is temporary. |
| 144 | |
| 145 | enum class FsType { |
| 146 | FILE, |
| 147 | DIRECTORY, |
| 148 | SYMLINK, |
| 149 | }; |
| 150 | |
| 151 | // Metadata about this filesystem node |
| 152 | struct Stat final { |
| 153 | FsType type = FsType::FILE; |
| 154 | |
| 155 | // The size of the node in bytes. For directories, this value will |
| 156 | // always be 0. Note that we are intentionally limiting the size of |
| 157 | // files to max(uint32_t) or 4GB. This is well above the maximum |
| 158 | // isolate heap limit so in-memory files should generally never |
| 159 | // get this large. |
| 160 | uint32_t size = 0; |
| 161 | |
| 162 | // The last modified time of the node. |
| 163 | kj::Date lastModified = kj::UNIX_EPOCH; |
| 164 | |
| 165 | // The creation time of the node. |
| 166 | kj::Date created = kj::UNIX_EPOCH; |
| 167 | |
| 168 | // Indicates if the node is writable. |
| 169 | bool writable = false; |
| 170 | |
| 171 | // Indicates if the node is a device. Certain operations |
| 172 | // may behave differently on devices. |
| 173 | bool device = false; |
| 174 | }; |
| 175 | |
| 176 | class SymbolicLink; |
| 177 | |
| 178 | enum class FsError { |
| 179 | // Path segment is not a directory |
| 180 | NOT_DIRECTORY, |
| 181 | // Directory is not empty |
| 182 | NOT_EMPTY, |
| 183 | // Node is read-only |
| 184 | READ_ONLY, |
| 185 | // Not permitted |
| 186 | NOT_PERMITTED, |
| 187 | // Not permitted on directory |
| 188 | NOT_PERMITTED_ON_DIRECTORY, |
| 189 | // Already exists |
| 190 | ALREADY_EXISTS, |
| 191 | // Too many open files |
| 192 | TOO_MANY_OPEN_FILES, |
| 193 | // Operation failed |
| 194 | FAILED, |
| 195 | // Not supported |
| 196 | NOT_SUPPORTED, |
| 197 | // Invalid path |
| 198 | INVALID_PATH, |
| 199 | // Exceeds file size limit |
| 200 | FILE_SIZE_LIMIT_EXCEEDED, |
| 201 | // Symlink depth exceeded |
| 202 | SYMLINK_DEPTH_EXCEEDED, |
| 203 | }; |
| 204 | |
| 205 | // A file in the virtual file system. If the file is read-only, then the |
| 206 | // mutation methods will throw an exception. |
| 207 | // |
| 208 | // A file is writable if it owns its contents. In such cases, the memory |
| 209 | // allocation is tracked by the corresponding isolate memory accounting. |
| 210 | class File: public kj::Refcounted { |
| 211 | public: |
| 212 | // Attempt to set the last modified time of this node. If the node is |
| 213 | // read-only, this will be a non-op. |
| 214 | virtual kj::Maybe<FsError> setLastModified(jsg::Lock& js, kj::Date date) = 0; |
| 215 | |
| 216 | // Returns the metadata for this node. |
| 217 | virtual Stat stat(jsg::Lock& js) KJ_WARN_UNUSED_RESULT = 0; |
| 218 | |
| 219 | // Reads all the contents of the file as a string. |
| 220 | kj::OneOf<FsError, jsg::JsString> readAllText(jsg::Lock& js) KJ_WARN_UNUSED_RESULT; |
| 221 | |
| 222 | // Reads all the contents of the file as a Uint8Array. |
| 223 | kj::OneOf<FsError, jsg::BufferSource> readAllBytes(jsg::Lock& js) KJ_WARN_UNUSED_RESULT; |
| 224 | |
| 225 | // Reads data from the file at the given offset into the given buffer. |
| 226 | virtual uint32_t read(jsg::Lock& js, uint32_t offset, kj::ArrayPtr<kj::byte> buffer) const = 0; |
| 227 | |
| 228 | // Replaces the full contents of the file with the given data. |
| 229 | // Equivalent to resize(js, data.size()) followed by write(js, 0, data). |
| 230 | kj::OneOf<FsError, uint32_t> writeAll( |
| 231 | jsg::Lock& js, kj::ArrayPtr<const kj::byte> data) KJ_WARN_UNUSED_RESULT { |
| 232 | KJ_IF_SOME(err, resize(js, data.size())) { |
| 233 | return err; |
| 234 | } |
| 235 | return write(js, 0, data); |
| 236 | } |
| 237 | |
| 238 | // Replaces the full contents of the file with the given data. |
| 239 | // Equivalent to resize(js, data.size()) followed by write(js, 0, data). |
| 240 | kj::OneOf<FsError, uint32_t> writeAll(jsg::Lock& js, kj::StringPtr data) KJ_WARN_UNUSED_RESULT { |
| 241 | return writeAll(js, data.asBytes()); |
| 242 | } |
| 243 | |
| 244 | // Writes data to the file at the given offset. Returns the number of bytes |
| 245 | // written. Returns the number of bytes written. If the file is not writable, |
| 246 | // this will throw an exception. If the offset is greater than the current |
| 247 | // size of the file, the file may be resized to accommodate the new data or |
| 248 | // an exception may be thrown (depending on the underlying implementation). |
| 249 | virtual kj::OneOf<FsError, uint32_t> write( |
| 250 | jsg::Lock& js, uint32_t offset, kj::ArrayPtr<const kj::byte> data) KJ_WARN_UNUSED_RESULT = 0; |
| 251 | |
| 252 | kj::OneOf<FsError, uint32_t> write( |
| 253 | jsg::Lock& js, uint32_t offset, kj::StringPtr data) KJ_WARN_UNUSED_RESULT { |
| 254 | return write(js, offset, data.asBytes()); |
| 255 | } |
| 256 | |
| 257 | // Fill the file with the given value from the given offset. This is more |
| 258 | // efficient that writing the same value as it does not require any allocations. |
| 259 | virtual kj::Maybe<FsError> fill( |
| 260 | jsg::Lock& js, kj::byte val, kj::Maybe<uint32_t> offset = kj::none) KJ_WARN_UNUSED_RESULT = 0; |
| 261 | |
| 262 | // Resize the file allocation. Note that this is potentially an expensive |
| 263 | // operation as it requires allocating a new internal buffer and copying |
| 264 | // the data. If the size is smaller than the current size, the contents of |
| 265 | // the fill will be truncated. If the size is larger than the current size, |
| 266 | // the new contents of the file will be filled with zeroes. |
| 267 | virtual kj::Maybe<FsError> resize(jsg::Lock& js, uint32_t size) KJ_WARN_UNUSED_RESULT = 0; |
| 268 | |
| 269 | // Creates a new readable/writable in-memory file. This file will not be |
| 270 | // initially included in a directory. To add it to a directory, use the |
| 271 | // add method. If the file is not added, it will be deleted |
| 272 | // when the handle is dropped. If size is given, the file will be initially |
| 273 | // filled with zeroes up to the given size, otherwise the file will be empty. |
| 274 | // The contents of the file will be tracked and counted towards the isolate |
| 275 | // external memory usage. |
| 276 | // If size is not given, the file will be empty. If the intent is to perform |
| 277 | // multiple writes to the file, it is recommended to specify a size up front |
| 278 | // or to resize the file as appropriate to account for the expected writes. |
| 279 | // This will avoid each individual write causing a reallocation of the |
| 280 | // internal buffer to accommodate the new data. |
| 281 | static kj::Rc<File> newWritable( |
| 282 | jsg::Lock& js, kj::Maybe<uint32_t> size = kj::none) KJ_WARN_UNUSED_RESULT; |
| 283 | |
| 284 | // Creates a new readable in-memory file wrapping the given data. The file |
| 285 | // does not take ownership of the data and the data must remain valid for the |
| 286 | // lifetime of the file. The file will be read-only. It will not be initially |
| 287 | // included in a directory. The contents of the file will not be tracked and |
| 288 | // will not count towards the isolate external memory usage. |
| 289 | static kj::Rc<File> newReadable(kj::ArrayPtr<const kj::byte> data) KJ_WARN_UNUSED_RESULT; |
| 290 | |
| 291 | virtual kj::StringPtr jsgGetMemoryName() const = 0; |
| 292 | virtual size_t jsgGetMemorySelfSize() const = 0; |
| 293 | virtual void jsgGetMemoryInfo(jsg::MemoryTracker& tracker) const = 0; |
| 294 | |
| 295 | // Creates a copy of this file. |
| 296 | virtual kj::OneOf<FsError, kj::Rc<File>> clone(jsg::Lock& js) KJ_WARN_UNUSED_RESULT = 0; |
| 297 | |
| 298 | // Replaces the contents of this file with the given file if possible. |
| 299 | // If this file is read-only, an exception will be thrown. |
| 300 | virtual kj::Maybe<FsError> replace(jsg::Lock& js, kj::Rc<File> file) KJ_WARN_UNUSED_RESULT = 0; |
| 301 | |
| 302 | // Returns a UUID that uniquely identifies this fs node. The value stable across |
| 303 | // multiple calls to getUniqueId() on the same node, but is not guaranteed to |
| 304 | // be stable across worker restarts. This is primarily useful for implementing |
| 305 | // the FileSystemHandle.getUniqueId() API, which is not yet fully standardized |
| 306 | // but is being implemented and has Web Platform Tests. |
| 307 | virtual kj::StringPtr getUniqueId(jsg::Lock&) const = 0; |
| 308 | |
| 309 | // Ensures that the symlink instance itself is counted towards the isolate |
| 310 | // memory limit. |
| 311 | virtual void countTowardsIsolateLimit(jsg::Lock& js) const { |
| 312 | // Non-op by default. |
| 313 | }; |
| 314 | }; |
| 315 | |
| 316 | // A directory in the virtual file system. If the directory is read-only, |
| 317 | // then the mutation methods will throw an exception. |
| 318 | class Directory: public kj::Refcounted { |
| 319 | public: |
| 320 | // Returns the metadata for this node. |
| 321 | virtual kj::Maybe<kj::OneOf<FsError, Stat>> stat( |
| 322 | jsg::Lock& js, kj::PathPtr ptr) KJ_WARN_UNUSED_RESULT = 0; |
| 323 | Stat stat(jsg::Lock& js) KJ_WARN_UNUSED_RESULT { |
| 324 | // In this case, stat is guaranteed to succeed, so skip the error checking. |
| 325 | return KJ_ASSERT_NONNULL(stat(js, nullptr)).get<Stat>(); |
| 326 | } |
| 327 | |
| 328 | // Return the number of entries in this directory. If typeFilter is provided, |
| 329 | // only entries matching the given type will be counted. |
| 330 | virtual size_t count( |
| 331 | jsg::Lock& js, kj::Maybe<FsType> typeFilter = kj::none) KJ_WARN_UNUSED_RESULT = 0; |
| 332 | |
| 333 | // Provides a simple iterator iterface over the directory entries. It is important |
| 334 | // to only use these iterators while holding the isolate lock as entries may have |
| 335 | // their refcounts incremented and decremented while iterating. To avoid issues using |
| 336 | // the iterators using idiomatic C++ syntax, we don't require passing the jsg::Lock& |
| 337 | // to the begin() and end() methods so it is important to ensure they are only called |
| 338 | // while holding the lock. |
| 339 | using Item = kj::OneOf<kj::Rc<File>, kj::Rc<Directory>, kj::Rc<SymbolicLink>>; |
| 340 | using Entry = kj::HashMap<kj::String, Item>::Entry; |
| 341 | virtual Entry* begin() = 0; |
| 342 | virtual Entry* end() = 0; |
| 343 | virtual const Entry* begin() const = 0; |
| 344 | virtual const Entry* end() const = 0; |
| 345 | |
| 346 | struct OpenOptions { |
| 347 | // When set, if the path does not exist, we will attempt to create the |
| 348 | // node as the specified type. Type must be one of either FILE or DIRECTORY. |
| 349 | // Specifying SYMBOLICLINK as the type will throw an exception. |
| 350 | kj::Maybe<FsType> createAs; |
| 351 | |
| 352 | // If the path points to a symbolic link, by default the link will be followed |
| 353 | // such that if the link points to a valid node, that node will be returned. |
| 354 | // If followLinks is false, however, the symbolic link itself will be returned. |
| 355 | bool followLinks = true; |
| 356 | }; |
| 357 | |
| 358 | // Tries opening the file or directory at the given path. If the node does |
| 359 | // not exist, and createAs is provided specifying a create mode, the node |
| 360 | // will be created as the specified type; otherwise kj::none is returned |
| 361 | // if the node does not exist. If the directory is read only and create is |
| 362 | // specified, an exception will be thrown. |
| 363 | // The path must be relative to the this directory. |
| 364 | // Note that when creating files using this method, the newly created file |
| 365 | // will have a size of 0 bytes. The file will be writable but each write |
| 366 | // could end up causing a new allocation. It the intent is to perform |
| 367 | // multiple writes, it is recommended to resize the file to the expected |
| 368 | // size up front or to use the newWritable method to create a file with the |
| 369 | // expected size and add it into the target directory. |
| 370 | // |
| 371 | // Note that tryOpen doubles as both an existence check and a stat operation. |
| 372 | // When the createAs parameter is not specified, the method will return |
| 373 | // kj::none if the file or directory does not exist. |
| 374 | // |
| 375 | // If the path identifies a symbolic link, the symbolic link will be resolved |
| 376 | // and the target, if it exists, will be returned. If the target does not exist, |
| 377 | // kj::none will be returned. |
| 378 | virtual kj::Maybe<kj::OneOf<FsError, kj::Rc<File>, kj::Rc<Directory>, kj::Rc<SymbolicLink>>> |
| 379 | tryOpen(jsg::Lock& js, |
| 380 | kj::PathPtr path, |
| 381 | OpenOptions options = {kj::none, true}) KJ_WARN_UNUSED_RESULT = 0; |
| 382 | |
| 383 | // Attempts to move the given file or directory this directory with the given |
| 384 | // name. If the name already exists an exception will be thrown. The name must |
| 385 | // not contain any path separators. If this directory is read only, an exception |
| 386 | // will be thrown. |
| 387 | virtual kj::Maybe<FsError> add( |
| 388 | jsg::Lock& js, kj::StringPtr name, Item entry) KJ_WARN_UNUSED_RESULT = 0; |
| 389 | |
| 390 | struct RemoveOptions { |
| 391 | // If true and the node is a directory, remove all entries in the directory. |
| 392 | bool recursive = false; |
| 393 | }; |
| 394 | |
| 395 | // Tries to remove the file, directory, or symlink at the given path. If the |
| 396 | // node does not exist, false will be returned. If the node is a directory and |
| 397 | // the recursive option is not set, an exception will be thrown if the directory |
| 398 | // is not empty. If the directory is read only, an exception will be thrown. |
| 399 | // If the node is a file, it will be removed regardless of the recursive option |
| 400 | // if the directory is not read only. |
| 401 | // If the node is a symlink, the symlink will be removed but the target will |
| 402 | // be left intact. |
| 403 | // The path must be relative to the current directory. |
| 404 | // Note that this method will only remove the node from this directory. If the |
| 405 | // node has references in other directories, those will not be removed. |
| 406 | // If the target is a symbolic link, the symbolic link will be removed but |
| 407 | // the target will not be removed. |
| 408 | virtual kj::OneOf<FsError, bool> remove( |
| 409 | jsg::Lock& js, kj::PathPtr path, RemoveOptions options = {false}) KJ_WARN_UNUSED_RESULT = 0; |
| 410 | |
| 411 | virtual kj::StringPtr jsgGetMemoryName() const = 0; |
| 412 | virtual size_t jsgGetMemorySelfSize() const = 0; |
| 413 | virtual void jsgGetMemoryInfo(jsg::MemoryTracker& tracker) const = 0; |
| 414 | |
| 415 | // Creates a new readable/writable in-memory directory. This directory will |
| 416 | // not be initially included in a directory. To add it to a directory, use the |
| 417 | // add method. If the directory is not added, it will be deleted |
| 418 | // when the handle is dropped. |
| 419 | static kj::Rc<Directory> newWritable() KJ_WARN_UNUSED_RESULT; |
| 420 | |
| 421 | // As a utility in some cases, we need the ability to create empty read-only |
| 422 | // directories. |
| 423 | static kj::Rc<Directory> newEmptyReadonly() KJ_WARN_UNUSED_RESULT; |
| 424 | |
| 425 | // Variation of newWritable that ensures the Directory instance itself is |
| 426 | // counted towwards the isolate memory limit. This should be the typical |
| 427 | // case for directories created by user code, such as when creating a |
| 428 | // temporary directory under /tmp. This should not be used for directories |
| 429 | // that are created by the runtime that aren't directly under the users |
| 430 | // control, such as the /tmp directory itself. |
| 431 | static kj::Rc<Directory> newWritable(jsg::Lock& js) KJ_WARN_UNUSED_RESULT; |
| 432 | |
| 433 | // Used to build a new read-only directory. All files and directories added |
| 434 | // may or may not be writable. The directory will not be initially included |
| 435 | // in a directory. To add it to a directory, use the add method. |
| 436 | class Builder { |
| 437 | public: |
| 438 | Builder() = default; |
| 439 | KJ_DISALLOW_COPY_AND_MOVE(Builder); |
| 440 | |
| 441 | // Adds a file or directory to the directory. The name must not contain any |
| 442 | // path separators. If the name already exists an exception is thrown. |
| 443 | void add(kj::StringPtr name, kj::OneOf<kj::Rc<File>, kj::Rc<Directory>> fileOrDirectory); |
| 444 | |
| 445 | // Adds a directory builder to the directory. This is used to incrementally build |
| 446 | // up read-only directories. When finish is called, the directory will be |
| 447 | // finalized. The name must not contain any path separators. |
| 448 | void add(kj::StringPtr name, kj::Own<Directory::Builder> dir); |
| 449 | |
| 450 | // Adds a file or directory at the given path. The path may contain path separators. |
| 451 | // Subdirectories will be created as needed (as read-only directories). |
| 452 | void addPath(kj::PathPtr path, kj::OneOf<kj::Rc<File>, kj::Rc<Directory>> fileOrDirectory); |
| 453 | |
| 454 | // Finalizes and returns the directory. The directory will not be writable. |
| 455 | // The directory will not be initially included in a directory. To add it |
| 456 | // to a directory, use the add method. |
| 457 | kj::Rc<Directory> finish() KJ_WARN_UNUSED_RESULT; |
| 458 | |
| 459 | using Map = kj::HashMap<kj::String, |
| 460 | kj::OneOf<kj::Rc<File>, kj::Rc<Directory>, kj::Own<Directory::Builder>>>; |
| 461 | using Entry = Map::Entry; |
| 462 | |
| 463 | private: |
| 464 | Map entries; |
| 465 | }; |
| 466 | |
| 467 | // Returns a UUID that uniquely identifies this fs node. The value stable across |
| 468 | // multiple calls to getUniqueId() on the same node, but is not guaranteed to |
| 469 | // be stable across worker restarts. This is primarily useful for implementing |
| 470 | // the FileSystemHandle.getUniqueId() API, which is not yet fully standardized |
| 471 | // but is being implemented and has Web Platform Tests. |
| 472 | virtual kj::StringPtr getUniqueId(jsg::Lock&) const = 0; |
| 473 | |
| 474 | // Ensures that the directory instance itself is counted towards the isolate |
| 475 | // memory limit. Not every directory needs to be counted so we don't do this |
| 476 | // by default automatically for every Directory instance. |
| 477 | virtual void countTowardsIsolateLimit(jsg::Lock& js) const { |
| 478 | // Non-op by default. |
| 479 | } |
| 480 | }; |
| 481 | |
| 482 | // The equivalent to a symbolic link. A symlink holds a reference to the |
| 483 | // virtual file system and a target path. The target node may or may not |
| 484 | // exist, can be deleted and recreated at any time and the symlink will |
| 485 | // remain valid. |
| 486 | class SymbolicLink final: public kj::Refcounted { |
| 487 | public: |
| 488 | SymbolicLink(kj::Rc<Directory> root, kj::Path targetPath) |
| 489 | : root(kj::mv(root)), |
| 490 | targetPath(kj::mv(targetPath)) {} |
| 491 | KJ_DISALLOW_COPY_AND_MOVE(SymbolicLink); |
| 492 | |
| 493 | // Gets the stat for the symbolic link itself. |
| 494 | Stat stat(jsg::Lock& js) KJ_WARN_UNUSED_RESULT; |
| 495 | |
| 496 | // Resolves the symbolic link into a file or directory. |
| 497 | kj::Maybe<kj::OneOf<FsError, kj::Rc<File>, kj::Rc<Directory>>> resolve( |
| 498 | jsg::Lock& js) KJ_WARN_UNUSED_RESULT; |
| 499 | |
| 500 | // Returns the target path of the symbolic link. |
| 501 | kj::PathPtr getTargetPath() const KJ_WARN_UNUSED_RESULT { |
| 502 | return targetPath; |
| 503 | } |
| 504 | |
| 505 | jsg::Url getTargetUrl() const KJ_WARN_UNUSED_RESULT; |
| 506 | |
| 507 | // Returns a UUID that uniquely identifies this fs node. The value stable across |
| 508 | // multiple calls to getUniqueId() on the same node, but is not guaranteed to |
| 509 | // be stable across worker restarts. This is primarily useful for implementing |
| 510 | // the FileSystemHandle.getUniqueId() API, which is not yet fully standardized |
| 511 | // but is being implemented and has Web Platform Tests. |
| 512 | kj::StringPtr getUniqueId(jsg::Lock&) const; |
| 513 | |
| 514 | // Ensures that the symlink instance itself is counted towards the isolate |
| 515 | // memory limit. |
| 516 | void countTowardsIsolateLimit(jsg::Lock& js) const; |
| 517 | |
| 518 | private: |
| 519 | kj::Rc<Directory> root; |
| 520 | kj::Path targetPath; |
| 521 | mutable kj::Maybe<kj::String> maybeUniqueId; |
| 522 | mutable kj::Maybe<jsg::ExternalMemoryAdjustment> maybeMemoryAdjustment; |
| 523 | }; |
| 524 | |
| 525 | using FsNode = kj::OneOf<kj::Rc<File>, kj::Rc<Directory>, kj::Rc<SymbolicLink>>; |
| 526 | using FsNodeWithError = kj::OneOf<FsError, kj::Rc<File>, kj::Rc<Directory>, kj::Rc<SymbolicLink>>; |
| 527 | class FsMap; |
| 528 | |
| 529 | // The virtual file system interface. This is the main entry point for accessing the vfs. |
| 530 | // It is important to always destroy the VirtualFileSystem instance under the isolate lock. |
| 531 | // The VFS holds a table of Refcounted objects (File, Directory, SymbolicLink) that can only |
| 532 | // be safely destroyed under the isolate lock. |
| 533 | class VirtualFileSystem { |
| 534 | public: |
| 535 | virtual ~VirtualFileSystem() noexcept(false) {} |
| 536 | |
| 537 | // The observer interface is used to allow the runtime to observe opening |
| 538 | // or closing of file descriptors in order to allow the runtime to keep |
| 539 | // an eye on the number of open file descriptors and take action if needed. |
| 540 | class Observer { |
| 541 | public: |
| 542 | virtual ~Observer() = default; |
| 543 | |
| 544 | // openFds is the number of currently open file descriptors, including |
| 545 | // the one that was just opened. totalFds is the total number of file |
| 546 | // descriptors that have been opened total. |
| 547 | virtual void onOpen(size_t openFds, size_t totalFds) const { |
| 548 | // By default, do nothing. |
| 549 | } |
| 550 | |
| 551 | // openFds is the number of currently open file descriptors, excluding |
| 552 | // the one that was just closed. totalFds is the total number of file |
| 553 | // descriptors that have been opened total. |
| 554 | virtual void onClose(size_t openFdCount, size_t totalFds) const { |
| 555 | // By default, do nothing. |
| 556 | } |
| 557 | |
| 558 | // Called when the total maximum number of file descriptors the worker |
| 559 | // is allowed to open is reached. After this point, the worker will no |
| 560 | // longer be allowed to open any new file descriptors. |
| 561 | virtual void onMaxFds(size_t openFdCount) const { |
| 562 | // By default, do nothing. |
| 563 | } |
| 564 | }; |
| 565 | |
| 566 | // The root of the virtual file system. |
| 567 | virtual kj::Rc<Directory> getRoot(jsg::Lock& js) const KJ_WARN_UNUSED_RESULT = 0; |
| 568 | |
| 569 | struct ResolveOptions { |
| 570 | bool followLinks = true; |
| 571 | }; |
| 572 | |
| 573 | // Resolves the given file URL into a file or directory. |
| 574 | kj::Maybe<FsNodeWithError> resolve(jsg::Lock& js, |
| 575 | const jsg::Url& url, |
| 576 | ResolveOptions options = {true}) const KJ_WARN_UNUSED_RESULT; |
| 577 | |
| 578 | // Resolves the given file URL into metadata for a file or directory. |
| 579 | kj::Maybe<kj::OneOf<FsError, Stat>> resolveStat( |
| 580 | jsg::Lock& js, const jsg::Url& url) const KJ_WARN_UNUSED_RESULT; |
| 581 | |
| 582 | // Creates a new symbolic link to the given target path. The target path |
| 583 | // does not need to exist. |
| 584 | kj::Rc<SymbolicLink> newSymbolicLink( |
| 585 | jsg::Lock& js, const jsg::Url& url) const KJ_WARN_UNUSED_RESULT; |
| 586 | |
| 587 | // Return the configured root paths for the bundle, temp, and dev directories. |
| 588 | virtual const jsg::Url& getBundleRoot() const KJ_WARN_UNUSED_RESULT = 0; |
| 589 | virtual const jsg::Url& getTmpRoot() const KJ_WARN_UNUSED_RESULT = 0; |
| 590 | virtual const jsg::Url& getDevRoot() const KJ_WARN_UNUSED_RESULT = 0; |
| 591 | |
| 592 | // Get the current virtual file system for the current isolate lock. |
| 593 | static const VirtualFileSystem& current(jsg::Lock&) KJ_WARN_UNUSED_RESULT; |
| 594 | |
| 595 | // ========================================================================== |
| 596 | // File Descriptor support |
| 597 | |
| 598 | struct OpenOptions { |
| 599 | // Open the file descriptor for reading. |
| 600 | bool read = true; |
| 601 | // Open the file descriptor for writing. |
| 602 | bool write = false; |
| 603 | // Open the file descriptor for appending. Ignored if write is false. |
| 604 | bool append = false; |
| 605 | |
| 606 | // If true, opening the path will fail if it already exists. |
| 607 | bool exclusive = false; |
| 608 | |
| 609 | // If true, and the destination is a symbolic link, the link will be |
| 610 | // followed such that the file descriptor is opened on the target |
| 611 | // of the symbolic link. If false, the file descriptor will be opened |
| 612 | // on the symbolic link itself. |
| 613 | bool followLinks = true; |
| 614 | }; |
| 615 | |
| 616 | // Represents an opened file descriptor. |
| 617 | struct OpenedFile: public kj::Refcounted { |
| 618 | // The file descriptor for the opened file. |
| 619 | int fd; |
| 620 | // The file descriptor was opened for reading. |
| 621 | bool read; |
| 622 | // The file descriptor was opened for writing. |
| 623 | bool write; |
| 624 | // The file descriptor was opened for appending (ignored if write is false). |
| 625 | bool append; |
| 626 | // The actual file, directory, or symlink that was opened. |
| 627 | FsNode node; |
| 628 | |
| 629 | OpenedFile(int fd, bool read, bool write, bool append, FsNode node) |
| 630 | : fd(fd), |
| 631 | read(read), |
| 632 | write(write), |
| 633 | append(append), |
| 634 | node(kj::mv(node)) {} |
| 635 | |
| 636 | // When reading from or writing to the file, if an offset is not |
| 637 | // explicitly given then the offset will be set to the current |
| 638 | // position in the file. |
| 639 | uint32_t position = 0; |
| 640 | }; |
| 641 | |
| 642 | enum class Stdio { |
| 643 | IN, |
| 644 | OUT, |
| 645 | ERR, |
| 646 | }; |
| 647 | virtual kj::Rc<OpenedFile> getStdio(jsg::Lock& js, Stdio stdio) const KJ_WARN_UNUSED_RESULT = 0; |
| 648 | |
| 649 | // Attempts to open a file descriptor for the given file URL. It's critical |
| 650 | // to understand that the file descriptor table is shared for the entire |
| 651 | // worker. This means that if a file descriptor is opened, it will remain |
| 652 | // open until it is closed or the worker is terminated. If too many file |
| 653 | // descriptors are left open the worker may run out of file descriptors and |
| 654 | // fail to open new files. Additionally, opening too many file descriptors |
| 655 | // may cause a production worker to be condemned in the system. There is a |
| 656 | // strict upper limit on the number of file descriptors that can be opened |
| 657 | // total. Once that limit is reached, all subsequent attempts to open a |
| 658 | // file descriptor will fail. |
| 659 | // |
| 660 | // If the file cannot be opened or created, an exception will be thrown. |
| 661 | virtual kj::OneOf<FsError, kj::Rc<OpenedFile>> openFd(jsg::Lock& js, |
| 662 | const jsg::Url& url, |
| 663 | OpenOptions options = {true, false, false, false, true}) const KJ_WARN_UNUSED_RESULT = 0; |
| 664 | |
| 665 | // Closes the given file descriptor. This is a no-op if the file descriptor is not open. |
| 666 | // Using an int fd is not super nice but it is the most compatible with the node:fs |
| 667 | // API and posix in general. |
| 668 | virtual void closeFd(jsg::Lock& js, int fd) const = 0; |
| 669 | |
| 670 | // Returns an opaque RAII handle that wraps a file descriptor. |
| 671 | // When it is dropped, the file descriptor will be closed. The handle |
| 672 | // holds a weak reference to the VirtualFileSystem so that, on the off |
| 673 | // chance the VirtualFileSystem is destroyed while the handle is still |
| 674 | // alive, destruction of the handle will become a no-op. |
| 675 | virtual kj::Own<void> wrapFd(jsg::Lock& js, int fd) const KJ_WARN_UNUSED_RESULT = 0; |
| 676 | |
| 677 | // Attempts to get the opened file, directory, or symlink for the given file descriptor. |
| 678 | // Returns kj::none if the fd is not opened/known. |
| 679 | virtual kj::Maybe<kj::Rc<OpenedFile>> tryGetFd( |
| 680 | jsg::Lock& js, int fd) const KJ_WARN_UNUSED_RESULT = 0; |
| 681 | |
| 682 | // Locks are used by the web file system to ensure that certain mutation operations |
| 683 | // are not permitted while a lock is held. These are not true locks, they are more |
| 684 | // like simple refcounts. While the refcount is greater than 0, the lock is held. |
| 685 | // The lock is released when the returned kj::Own<void> is dropped. |
| 686 | virtual kj::Own<void> lock( |
| 687 | jsg::Lock& js, const jsg::Url& locator) const KJ_WARN_UNUSED_RESULT = 0; |
| 688 | virtual bool isLocked(jsg::Lock& js, const jsg::Url& locator) const KJ_WARN_UNUSED_RESULT = 0; |
| 689 | }; |
| 690 | |
| 691 | kj::Own<VirtualFileSystem> newVirtualFileSystem(kj::Own<FsMap> fsMap, |
| 692 | kj::Rc<Directory>&& root, |
| 693 | kj::Own<VirtualFileSystem::Observer> observer = kj::heap<VirtualFileSystem::Observer>()) |
| 694 | KJ_WARN_UNUSED_RESULT; |
| 695 | |
| 696 | // The FsMap is a configurable mapping of built-in "known" file system |
| 697 | // paths to user-configurable locations. It is used to allow user-specified |
| 698 | // "mount" points for certain built-in file system paths. For example, the |
| 699 | // WorkerFileSystem exposes a built-in bundle path that is located at /bundle |
| 700 | // by default. The FsMap allows the user to specify a different name for this |
| 701 | // path, such as /mybundle, or even nested paths like /mybundle/a/b/c/modules/. |
| 702 | // To add a new known root, add the name to the KNOWN_VFS_ROOTS macro and |
| 703 | // specify the default path. The relevant API methods on the FsMap class |
| 704 | // will be generated via the template. |
| 705 | #define KNOWN_VFS_ROOTS(V) \ |
| 706 | V(Bundle, "file:///bundle/") \ |
| 707 | V(Temp, "file:///tmp/") \ |
| 708 | V(Dev, "file:///dev/") |
| 709 | |
| 710 | class FsMap final { |
| 711 | public: |
| 712 | FsMap() = default; |
| 713 | KJ_DISALLOW_COPY_AND_MOVE(FsMap); |
| 714 | |
| 715 | #define DEFINE_METHODS_FOR_ROOTS(name, _) \ |
| 716 | static const jsg::Url kDefault##name##Path; \ |
| 717 | const jsg::Url& get##name##Root() const { \ |
| 718 | return maybe##name##Root.orDefault(kDefault##name##Path); \ |
| 719 | } \ |
| 720 | const kj::Path get##name##Path() const { \ |
| 721 | auto path = kj::str(get##name##Root().getPathname().slice(1)); \ |
| 722 | return kj::Path::parse(path); \ |
| 723 | } \ |
| 724 | void set##name##Root(jsg::Url url) { \ |
| 725 | KJ_REQUIRE(url.getProtocol() == "file:"_kj, "url must be a file URL"); \ |
| 726 | maybe##name##Root = kj::mv(url); \ |
| 727 | } \ |
| 728 | void set##name##Root(kj::StringPtr path) { \ |
| 729 | auto url = KJ_REQUIRE_NONNULL(jsg::Url::tryParse(path, "file:///"_kj), "invalid path"); \ |
| 730 | set##name##Root(kj::mv(url)); \ |
| 731 | } |
| 732 | |
| 733 | KNOWN_VFS_ROOTS(DEFINE_METHODS_FOR_ROOTS) |
| 734 | |
| 735 | #undef DEFINE_METHODS_FOR_ROOTS |
| 736 | |
| 737 | private: |
| 738 | #define DEFINE_FIELDS_FOR_ROOTS(name, _) kj::Maybe<jsg::Url> maybe##name##Root; |
| 739 | KNOWN_VFS_ROOTS(DEFINE_FIELDS_FOR_ROOTS) |
| 740 | #undef DEFINE_FIELDS_FOR_ROOTS |
| 741 | }; |
| 742 | |
| 743 | // An RAII object that stores the temporary directory items. |
| 744 | // Instances can either live on the stack (in which case they |
| 745 | // will be set in a thread-local and hasCurrent() will return |
| 746 | // true, or they can be created on the heap and held (e.g. in |
| 747 | // IoContext). |
| 748 | class TmpDirStoreScope final { |
| 749 | public: |
| 750 | static bool hasCurrent(); |
| 751 | static TmpDirStoreScope& current(); |
| 752 | TmpDirStoreScope(kj::Maybe<kj::Badge<TmpDirStoreScope>> guard = kj::none); |
| 753 | KJ_DISALLOW_COPY_AND_MOVE(TmpDirStoreScope); |
| 754 | ~TmpDirStoreScope() noexcept(false); |
| 755 | |
| 756 | static kj::Own<TmpDirStoreScope> create(); |
| 757 | |
| 758 | kj::Rc<Directory> getDirectory() const { |
| 759 | return dir.addRef(); |
| 760 | } |
| 761 | |
| 762 | kj::PathPtr getCwd() const { |
| 763 | return kj::PathPtr(cwd); |
| 764 | } |
| 765 | |
| 766 | void setCwd(kj::Path newCwd) { |
| 767 | cwd = kj::mv(newCwd); |
| 768 | } |
| 769 | |
| 770 | private: |
| 771 | mutable kj::Rc<Directory> dir; |
| 772 | kj::Path cwd; |
| 773 | bool onStack = false; |
| 774 | }; |
| 775 | |
| 776 | // A scope utility that is used to guard against infinite recursion when |
| 777 | // resolving symbolic links. This should only ever be stack allocated and |
| 778 | // should never be shared outside of the current execution scope. |
| 779 | class SymbolicLinkRecursionGuardScope final { |
| 780 | public: |
| 781 | SymbolicLinkRecursionGuardScope(); |
| 782 | ~SymbolicLinkRecursionGuardScope() noexcept(false); |
| 783 | KJ_DISALLOW_COPY_AND_MOVE(SymbolicLinkRecursionGuardScope); |
| 784 | |
| 785 | // Whenever a symbolic link is resolved, it needs to be checked against |
| 786 | // the recursion guard. If the link has already been seen, an exception |
| 787 | // will be thrown. If the link has not been seen, it will be recorded |
| 788 | // as seen so that it can be checked against in the future. |
| 789 | static kj::Maybe<FsError> checkSeen(SymbolicLink* link) KJ_WARN_UNUSED_RESULT; |
| 790 | |
| 791 | private: |
| 792 | kj::HashSet<SymbolicLink*> linksSeen; |
| 793 | }; |
| 794 | |
| 795 | // Every Worker instance has its own virtual filesystem. At a minimum, this |
| 796 | // filesystem contains the worker's own bundled modules/files and a temporary |
| 797 | // in-memory directory for the worker to use. The filesystem is not shared |
| 798 | // between workers. The bundle delegate is a virtual directory delegate that |
| 799 | // provides the directory structure for the worker's bundle. |
| 800 | kj::Own<VirtualFileSystem> newWorkerFileSystem(kj::Own<FsMap> fsMap, |
| 801 | kj::Rc<Directory> bundleDirectory, |
| 802 | kj::Own<VirtualFileSystem::Observer> observer = kj::heap<VirtualFileSystem::Observer>()) |
| 803 | KJ_WARN_UNUSED_RESULT; |
| 804 | |
| 805 | // Exposed only for testing purposes. |
| 806 | kj::Rc<Directory> getTmpDirectoryImpl() KJ_WARN_UNUSED_RESULT; |
| 807 | |
| 808 | // Returns a directory that is lazily loaded on first access. |
| 809 | kj::Rc<Directory> getLazyDirectoryImpl( |
| 810 | kj::Function<kj::Rc<Directory>()> func) KJ_WARN_UNUSED_RESULT; |
| 811 | |
| 812 | kj::Rc<File> getDevNull() KJ_WARN_UNUSED_RESULT; |
| 813 | kj::Rc<File> getDevZero() KJ_WARN_UNUSED_RESULT; |
| 814 | kj::Rc<File> getDevFull() KJ_WARN_UNUSED_RESULT; |
| 815 | kj::Rc<File> getDevRandom() KJ_WARN_UNUSED_RESULT; |
| 816 | kj::Rc<Directory> getDevDirectory() KJ_WARN_UNUSED_RESULT; |
| 817 | |
| 818 | // Helper functions for current working directory management |
| 819 | kj::Maybe<kj::PathPtr> getCurrentWorkingDirectory() KJ_WARN_UNUSED_RESULT; |
| 820 | bool setCurrentWorkingDirectory(kj::Path newCwd) KJ_WARN_UNUSED_RESULT; |
| 821 | |
| 822 | } // namespace workerd |