mirror of
https://github.com/Dan-314/ravenswood-archive-stacks.git
synced 2026-05-18 08:04:04 -05:00
inital commit
This commit is contained in:
Dan-314
@@ -0,0 +1,156 @@
|
||||
name: Sync Scripts
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * *' # Daily at 3am UTC
|
||||
workflow_dispatch: # Manual trigger
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Sync scripts from all sources
|
||||
run: |
|
||||
node -e "
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
|
||||
function fetch(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const mod = url.startsWith('https') ? https : http;
|
||||
mod.get(url, res => {
|
||||
let data = '';
|
||||
res.on('data', chunk => data += chunk);
|
||||
res.on('end', () => {
|
||||
if (res.statusCode !== 200) return reject(new Error('HTTP ' + res.statusCode));
|
||||
resolve(JSON.parse(data));
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function syncSource(slug, manifest) {
|
||||
if (manifest.sync_method !== 'github_action' || !manifest.api_endpoint) {
|
||||
console.log('Skipping ' + slug + ' (sync_method=' + manifest.sync_method + ')');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Syncing from ' + slug + ' (' + manifest.api_endpoint + ')...');
|
||||
let cursor = null;
|
||||
let total = 0;
|
||||
|
||||
while (true) {
|
||||
let url = manifest.api_endpoint + '?limit=100';
|
||||
if (cursor) url += '&cursor=' + cursor;
|
||||
|
||||
let response;
|
||||
try { response = await fetch(url); }
|
||||
catch (e) { console.error('Failed to fetch from ' + slug + ': ' + e.message); break; }
|
||||
|
||||
for (const script of response.scripts) {
|
||||
// Write script file using JSON.stringify (matches Postgres jsonb::text format)
|
||||
const content = JSON.stringify(script.raw_json);
|
||||
fs.writeFileSync(path.join('scripts', script.hash + '.json'), content);
|
||||
|
||||
// Update source manifest
|
||||
manifest.scripts[script.hash] = {
|
||||
source_id: script.source_id,
|
||||
source_url: script.source_url,
|
||||
synced_at: new Date().toISOString()
|
||||
};
|
||||
total++;
|
||||
}
|
||||
|
||||
cursor = response.next_cursor;
|
||||
if (!cursor) break;
|
||||
}
|
||||
|
||||
// Write updated manifest
|
||||
fs.writeFileSync(
|
||||
path.join('sources', slug + '.json'),
|
||||
JSON.stringify(manifest, null, 2)
|
||||
);
|
||||
console.log('Done syncing ' + slug + ': ' + total + ' script(s)');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sourcesDir = 'sources';
|
||||
for (const f of fs.readdirSync(sourcesDir)) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
const slug = f.replace('.json', '');
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(sourcesDir, f), 'utf8'));
|
||||
await syncSource(slug, manifest);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => { console.error(e); process.exit(1); });
|
||||
"
|
||||
|
||||
- name: Rebuild index
|
||||
run: |
|
||||
node -e "
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const scriptsDir = 'scripts';
|
||||
const sourcesDir = 'sources';
|
||||
const index = [];
|
||||
|
||||
// Load all source manifests
|
||||
const sources = {};
|
||||
for (const f of fs.readdirSync(sourcesDir)) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
const slug = f.replace('.json', '');
|
||||
sources[slug] = JSON.parse(fs.readFileSync(path.join(sourcesDir, f), 'utf8'));
|
||||
}
|
||||
|
||||
// Build index from script files
|
||||
for (const f of fs.readdirSync(scriptsDir)) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
const hash = f.replace('.json', '');
|
||||
const script = JSON.parse(fs.readFileSync(path.join(scriptsDir, f), 'utf8'));
|
||||
const meta = script.find(e => e && e.id === '_meta') || {};
|
||||
const chars = script.filter(e => typeof e === 'string' || (e && e.id !== '_meta'));
|
||||
|
||||
// Find which sources have this hash
|
||||
const scriptSources = Object.entries(sources)
|
||||
.filter(([, s]) => s.scripts && s.scripts[hash])
|
||||
.map(([slug]) => slug);
|
||||
|
||||
index.push({
|
||||
hash,
|
||||
name: meta.name || 'Untitled',
|
||||
author: meta.author || null,
|
||||
character_count: chars.length,
|
||||
sources: scriptSources
|
||||
});
|
||||
}
|
||||
|
||||
index.sort((a, b) => a.name.localeCompare(b.name));
|
||||
fs.writeFileSync('index.json', JSON.stringify(index, null, 2));
|
||||
console.log('Index rebuilt with ' + index.length + ' scripts');
|
||||
"
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "archive-bot"
|
||||
git config user.email "bot@ravenswoodarchive.com"
|
||||
git add scripts/ sources/ index.json
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
count=$(git diff --cached --name-only -- scripts/ | wc -l)
|
||||
git commit -m "sync: ${count} script(s) updated"
|
||||
git push
|
||||
fi
|
||||
@@ -0,0 +1,117 @@
|
||||
name: Validate PR
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm install ajv ajv-formats
|
||||
|
||||
- name: Validate changed scripts
|
||||
run: |
|
||||
node -e "
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const Ajv = require('ajv');
|
||||
|
||||
const ajv = new Ajv({ allErrors: true });
|
||||
const schema = JSON.parse(fs.readFileSync('schema/script.schema.json', 'utf8'));
|
||||
const validate = ajv.compile(schema);
|
||||
|
||||
// Get changed files from git
|
||||
const { execSync } = require('child_process');
|
||||
const changed = execSync('git diff --name-only --diff-filter=ACMR origin/main...HEAD -- scripts/')
|
||||
.toString().trim().split('\n').filter(Boolean);
|
||||
|
||||
if (changed.length === 0) {
|
||||
console.log('No script files changed');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let errors = 0;
|
||||
|
||||
for (const file of changed) {
|
||||
const hash = file.replace('scripts/', '').replace('.json', '');
|
||||
const content = fs.readFileSync(file, 'utf8');
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch (e) {
|
||||
console.error('FAIL [' + file + ']: invalid JSON');
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Re-serialize through JSON.stringify to match DB hash format (Postgres jsonb::text)
|
||||
const canonical = JSON.stringify(parsed);
|
||||
const actualHash = crypto.createHash('md5').update(canonical).digest('hex');
|
||||
|
||||
// Check filename matches content hash
|
||||
if (hash !== actualHash) {
|
||||
console.error('FAIL [' + file + ']: filename hash mismatch. Expected ' + actualHash + ', got ' + hash);
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const script = parsed;
|
||||
|
||||
if (!validate(script)) {
|
||||
console.error('FAIL [' + file + ']: schema validation failed');
|
||||
for (const err of validate.errors) {
|
||||
console.error(' ' + err.instancePath + ' ' + err.message);
|
||||
}
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check _meta block exists
|
||||
const meta = script.find(e => e && e.id === '_meta');
|
||||
if (!meta) {
|
||||
console.error('FAIL [' + file + ']: missing _meta block');
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log('PASS [' + file + ']: ' + meta.name);
|
||||
}
|
||||
|
||||
if (errors > 0) {
|
||||
console.error(errors + ' file(s) failed validation');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('All ' + changed.length + ' file(s) passed validation');
|
||||
"
|
||||
|
||||
- name: Check source manifest integrity
|
||||
run: |
|
||||
node -e "
|
||||
const { execSync } = require('child_process');
|
||||
const changed = execSync('git diff --name-only --diff-filter=ACMR origin/main...HEAD -- sources/')
|
||||
.toString().trim().split('\n').filter(Boolean);
|
||||
|
||||
if (changed.length === 0) {
|
||||
console.log('No source manifests changed');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// PR author should only modify their own source manifest
|
||||
if (changed.length > 1) {
|
||||
console.error('FAIL: PR modifies multiple source manifests: ' + changed.join(', '));
|
||||
console.error('Each contributor should only modify their own sources/{site}.json');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('Source manifest check passed: ' + changed[0]);
|
||||
"
|
||||
Reference in New Issue
Block a user