admin/szjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
波澜壮阔
2025-03-07 22:27:18 +08:00
commit 912da26042
4457 changed files with 306818 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
node_modules/file-entry-cache/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) Roy Riojas & Jared Wray
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

116
node_modules/file-entry-cache/README.md generated vendored Normal file
View File

@@ -0,0 +1,116 @@
# file-entry-cache
> Super simple cache for file metadata, useful for process that work on a given series of files and that only need to repeat the job on the changed ones since the previous run of the process
[![NPM Version](https://img.shields.io/npm/v/file-entry-cache.svg?style=flat)](https://npmjs.org/package/file-entry-cache)
[![tests](https://github.com/jaredwray/file-entry-cache/actions/workflows/tests.yaml/badge.svg?branch=master)](https://github.com/jaredwray/file-entry-cache/actions/workflows/tests.yaml)
[![codecov](https://codecov.io/github/jaredwray/file-entry-cache/graph/badge.svg?token=37tZMQE0Sy)](https://codecov.io/github/jaredwray/file-entry-cache)
[![npm](https://img.shields.io/npm/dm/file-entry-cache)](https://npmjs.com/package/file-entry-cache)
## install
```bash
npm i --save file-entry-cache
```
## Usage
The module exposes two functions `create` and `createFromFile`.
## `create(cacheName, [directory, useCheckSum, currentWorkingDir])`
- **cacheName**: the name of the cache to be created
- **directory**: Optional the directory to load the cache from
- **usecheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
- **currentWorkingDir**: Optional the current working directory to use when resolving relative paths
## `createFromFile(pathToCache, [useCheckSum, currentWorkingDir])`
- **pathToCache**: the path to the cache file (this combines the cache name and directory)
- **useCheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
- **currentWorkingDir**: Optional the current working directory to use when resolving relative paths
```js
// loads the cache, if one does not exists for the given
// Id a new one will be prepared to be created
var fileEntryCache = require('file-entry-cache');
var cache = fileEntryCache.create('testCache');
var files = expand('../fixtures/*.txt');
// the first time this method is called, will return all the files
var oFiles = cache.getUpdatedFiles(files);
// this will persist this to disk checking each file stats and
// updating the meta attributes `size` and `mtime`.
// custom fields could also be added to the meta object and will be persisted
// in order to retrieve them later
cache.reconcile();
// use this if you want the non visited file entries to be kept in the cache
// for more than one execution
//
// cache.reconcile( true /* noPrune */)
// on a second run
var cache2 = fileEntryCache.create('testCache');
// will return now only the files that were modified or none
// if no files were modified previous to the execution of this function
var oFiles = cache.getUpdatedFiles(files);
// if you want to prevent a file from being considered non modified
// something useful if a file failed some sort of validation
// you can then remove the entry from the cache doing
cache.removeEntry('path/to/file'); // path to file should be the same path of the file received on `getUpdatedFiles`
// that will effectively make the file to appear again as modified until the validation is passed. In that
// case you should not remove it from the cache
// if you need all the files, so you can determine what to do with the changed ones
// you can call
var oFiles = cache.normalizeEntries(files);
// oFiles will be an array of objects like the following
entry = {
key: 'some/name/file', the path to the file
changed: true, // if the file was changed since previous run
meta: {
size: 3242, // the size of the file
mtime: 231231231, // the modification time of the file
data: {} // some extra field stored for this file (useful to save the result of a transformation on the file
}
}
```
## Motivation for this module
I needed a super simple and dumb **in-memory cache** with optional disk persistence (write-back cache) in order to make
a script that will beautify files with `esformatter` to execute only on the files that were changed since the last run.
In doing so the process of beautifying files was reduced from several seconds to a small fraction of a second.
This module uses [flat-cache](https://www.npmjs.com/package/flat-cache) a super simple `key/value` cache storage with
optional file persistance.
The main idea is to read the files when the task begins, apply the transforms required, and if the process succeed,
then store the new state of the files. The next time this module request for `getChangedFiles` will return only
the files that were modified. Making the process to end faster.
This module could also be used by processes that modify the files applying a transform, in that case the result of the
transform could be stored in the `meta` field, of the entries. Anything added to the meta field will be persisted.
Those processes won't need to call `getChangedFiles` they will instead call `normalizeEntries` that will return the
entries with a `changed` field that can be used to determine if the file was changed or not. If it was not changed
the transformed stored data could be used instead of actually applying the transformation, saving time in case of only
a few files changed.
In the worst case scenario all the files will be processed. In the best case scenario only a few of them will be processed.
## Important notes
- The values set on the meta attribute of the entries should be `stringify-able` ones if possible, flat-cache uses `circular-json` to try to persist circular structures, but this should be considered experimental. The best results are always obtained with non circular values
- All the changes to the cache state are done to memory first and only persisted after reconcile.
## License
MIT (c) Jared Wray

318
node_modules/file-entry-cache/cache.js generated vendored Normal file
View File

@@ -0,0 +1,318 @@
/* eslint-disable unicorn/no-this-assignment, func-names, no-multi-assign */
const path = require('node:path');
const crypto = require('node:crypto');
module.exports = {
createFromFile(filePath, useChecksum, currentWorkingDir) {
const fname = path.basename(filePath);
const dir = path.dirname(filePath);
return this.create(fname, dir, useChecksum, currentWorkingDir);
},
create(cacheId, _path, useChecksum, currentWorkingDir) {
const fs = require('node:fs');
const flatCache = require('flat-cache');
const cache = flatCache.load(cacheId, _path);
let normalizedEntries = {};
const removeNotFoundFiles = function removeNotFoundFiles() {
const cachedEntries = cache.keys();
// Remove not found entries
for (const fPath of cachedEntries) {
try {
let filePath = fPath;
if (currentWorkingDir) {
filePath = path.join(currentWorkingDir, fPath);
}
fs.statSync(filePath);
} catch (error) {
if (error.code === 'ENOENT') {
cache.removeKey(fPath);
}
}
}
};
removeNotFoundFiles();
return {
/**
* The flat cache storage used to persist the metadata of the `files
* @type {Object}
*/
cache,
/**
* To enable relative paths as the key with current working directory
* @type {string}
*/
currentWorkingDir: currentWorkingDir ?? undefined,
/**
* Given a buffer, calculate md5 hash of its content.
* @method getHash
* @param {Buffer} buffer buffer to calculate hash on
* @return {String} content hash digest
*/
getHash(buffer) {
return crypto.createHash('md5').update(buffer).digest('hex');
},
/**
* Return whether or not a file has changed since last time reconcile was called.
* @method hasFileChanged
* @param {String} file the filepath to check
* @return {Boolean} wheter or not the file has changed
*/
hasFileChanged(file) {
return this.getFileDescriptor(file).changed;
},
/**
* Given an array of file paths it return and object with three arrays:
* - changedFiles: Files that changed since previous run
* - notChangedFiles: Files that haven't change
* - notFoundFiles: Files that were not found, probably deleted
*
* @param {Array} files the files to analyze and compare to the previous seen files
* @return {[type]} [description]
*/
analyzeFiles(files) {
const me = this;
files ||= [];
const res = {
changedFiles: [],
notFoundFiles: [],
notChangedFiles: [],
};
for (const entry of me.normalizeEntries(files)) {
if (entry.changed) {
res.changedFiles.push(entry.key);
continue;
}
if (entry.notFound) {
res.notFoundFiles.push(entry.key);
continue;
}
res.notChangedFiles.push(entry.key);
}
return res;
},
getFileDescriptor(file) {
let fstat;
try {
fstat = fs.statSync(file);
} catch (error) {
this.removeEntry(file);
return {key: file, notFound: true, err: error};
}
if (useChecksum) {
return this._getFileDescriptorUsingChecksum(file);
}
return this._getFileDescriptorUsingMtimeAndSize(file, fstat);
},
_getFileKey(file) {
if (this.currentWorkingDir) {
return file.split(this.currentWorkingDir).pop();
}
return file;
},
_getFileDescriptorUsingMtimeAndSize(file, fstat) {
let meta = cache.getKey(this._getFileKey(file));
const cacheExists = Boolean(meta);
const cSize = fstat.size;
const cTime = fstat.mtime.getTime();
let isDifferentDate;
let isDifferentSize;
if (meta) {
isDifferentDate = cTime !== meta.mtime;
isDifferentSize = cSize !== meta.size;
} else {
meta = {size: cSize, mtime: cTime};
}
const nEntry = (normalizedEntries[this._getFileKey(file)] = {
key: this._getFileKey(file),
changed: !cacheExists || isDifferentDate || isDifferentSize,
meta,
});
return nEntry;
},
_getFileDescriptorUsingChecksum(file) {
let meta = cache.getKey(this._getFileKey(file));
const cacheExists = Boolean(meta);
let contentBuffer;
try {
contentBuffer = fs.readFileSync(file);
} catch {
contentBuffer = '';
}
let isDifferent = true;
const hash = this.getHash(contentBuffer);
if (meta) {
isDifferent = hash !== meta.hash;
} else {
meta = {hash};
}
const nEntry = (normalizedEntries[this._getFileKey(file)] = {
key: this._getFileKey(file),
changed: !cacheExists || isDifferent,
meta,
});
return nEntry;
},
/**
* Return the list o the files that changed compared
* against the ones stored in the cache
*
* @method getUpdated
* @param files {Array} the array of files to compare against the ones in the cache
* @returns {Array}
*/
getUpdatedFiles(files) {
const me = this;
files ||= [];
return me
.normalizeEntries(files)
.filter(entry => entry.changed)
.map(entry => entry.key);
},
/**
* Return the list of files
* @method normalizeEntries
* @param files
* @returns {*}
*/
normalizeEntries(files) {
files ||= [];
const me = this;
const nEntries = files.map(file => me.getFileDescriptor(file));
// NormalizeEntries = nEntries;
return nEntries;
},
/**
* Remove an entry from the file-entry-cache. Useful to force the file to still be considered
* modified the next time the process is run
*
* @method removeEntry
* @param entryName
*/
removeEntry(entryName) {
delete normalizedEntries[this._getFileKey(entryName)];
cache.removeKey(this._getFileKey(entryName));
},
/**
* Delete the cache file from the disk
* @method deleteCacheFile
*/
deleteCacheFile() {
cache.removeCacheFile();
},
/**
* Remove the cache from the file and clear the memory cache
*/
destroy() {
normalizedEntries = {};
cache.destroy();
},
_getMetaForFileUsingCheckSum(cacheEntry) {
let filePath = cacheEntry.key;
if (this.currentWorkingDir) {
filePath = path.join(this.currentWorkingDir, filePath);
}
const contentBuffer = fs.readFileSync(filePath);
const hash = this.getHash(contentBuffer);
const meta = Object.assign(cacheEntry.meta, {hash});
delete meta.size;
delete meta.mtime;
return meta;
},
_getMetaForFileUsingMtimeAndSize(cacheEntry) {
let filePath = cacheEntry.key;
if (currentWorkingDir) {
filePath = path.join(currentWorkingDir, filePath);
}
const stat = fs.statSync(filePath);
const meta = Object.assign(cacheEntry.meta, {
size: stat.size,
mtime: stat.mtime.getTime(),
});
delete meta.hash;
return meta;
},
/**
* Sync the files and persist them to the cache
* @method reconcile
*/
reconcile(noPrune) {
removeNotFoundFiles();
noPrune = noPrune === undefined ? true : noPrune;
const entries = normalizedEntries;
const keys = Object.keys(entries);
if (keys.length === 0) {
return;
}
const me = this;
for (const entryName of keys) {
const cacheEntry = entries[entryName];
try {
const meta = useChecksum
? me._getMetaForFileUsingCheckSum(cacheEntry)
: me._getMetaForFileUsingMtimeAndSize(cacheEntry);
cache.setKey(this._getFileKey(entryName), meta);
} catch (error) {
// If the file does not exists we don't save it
// other errors are just thrown
if (error.code !== 'ENOENT') {
throw error;
}
}
}
cache.save(noPrune);
},
};
},
};

59
node_modules/file-entry-cache/package.json generated vendored Normal file
View File

@@ -0,0 +1,59 @@
{
"name": "file-entry-cache",
"version": "9.1.0",
"description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
"repository": "jaredwray/file-entry-cache",
"license": "MIT",
"author": {
"name": "Jared Wray",
"url": "https://jaredwray.com"
},
"main": "cache.js",
"files": [
"cache.js"
],
"engines": {
"node": ">=18"
},
"scripts": {
"clean": "rimraf ./coverage /node_modules ./package-lock.json ./yarn.lock ./pnpm-lock.yaml",
"test": "xo --fix && c8 mocha -R spec test/specs/cache.js test/relative.js",
"test:relative": "rimraf ./rfixtures ./tfixtures && mocha test/relative.js",
"test:ci": "xo && c8 --reporter=lcov mocha -R spec test/specs/cache.js test/relative.js",
"perf": "node perf.js"
},
"prepush": [
"npm run test"
],
"precommit": [
"npm run test"
],
"keywords": [
"file cache",
"task cache files",
"file cache",
"key par",
"key value",
"cache"
],
"devDependencies": {
"c8": "^10.1.2",
"chai": "^4.3.10",
"glob-expand": "^0.2.1",
"mocha": "^10.5.1",
"rimraf": "^5.0.7",
"webpack": "^5.92.1",
"write": "^2.0.0",
"xo": "^0.58.0"
},
"dependencies": {
"flat-cache": "^5.0.0"
},
"xo": {
"rules": {
"unicorn/prefer-module": "off",
"n/prefer-global/process": "off",
"unicorn/prevent-abbreviations": "off"
}
}
}