add yarn script to extract masterlist

registry/.gitignore

@@ -13,6 +13,7 @@ outputs/plain_text/*
 outputs/plain_text_master/*
 outputs/plain_text_unique/*
 outputs/unique_pem
+outputs/csca/
 src/csca/python scripts/__pycache__
 dsc_plain_text
 

registry/package.json

@@ -17,5 +17,9 @@
     "node-rsa": "^1.1.1",
     "pkijs": "^3.2.4",
     "poseidon-lite": "^0.2.0"
-  }
-}
+  },
+  "scripts": {
+    "extract": "ts-node src/extractMasterList.ts"
+  },
+  "license": "MIT"
+}

registry/src/csca/extract_masterlist_csca.ts

@@ -2,109 +2,110 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 
-// extract masterlists from ICAO ldif file
-const fileContent = fs.readFileSync("inputs/icao_download_section/icaopkd-002-complete-000243.ldif", "utf-8");
-const regex = /pkdMasterListContent::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
-let match: RegExpExecArray | null;
+export async function extractMasterlistCsca() {
+  // Extract masterlists from ICAO ldif file
+  const ldif_path = path.join(__dirname, '..', '..', 'inputs', 'icao_download_section', 'icaopkd-002-complete-000243.ldif');
+  const masterlist_path = path.join(__dirname, '..', '..', 'outputs', 'masterlists');
+  const csca_path = path.join(__dirname, '..', '..', 'outputs', 'csca');
 
-const masterlists: string[] = [];
+  const fileContent = fs.readFileSync(ldif_path, "utf-8");
+  const regex = /pkdMasterListContent::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
+  let match: RegExpExecArray | null;
 
-while ((match = regex.exec(fileContent)) !== null) {
-  const masterlist = match[1].replace(/\s+/g, "");
-  masterlists.push(masterlist);
-}
+  const masterlists: string[] = [];
 
-if (!fs.existsSync("outputs/masterlists/")) {
-  fs.mkdirSync("outputs/masterlists/");
-}
-
-for (let i = 0; i < masterlists.length; i++) {
-  fs.writeFileSync(
-    path.join("outputs/masterlists/", `masterlist_${i}.pem`),
-    `-----BEGIN CERTIFICATE-----\n${masterlists[i]}\n-----END CERTIFICATE-----\n`
-  );
-}
-
-console.log(`Extracted ${masterlists.length} masterlists.`);
-
-for (let i = 0; i < masterlists.length; i++) {
-  execSync(`openssl asn1parse -in outputs/masterlists/masterlist_${i}.pem -inform PEM -i > outputs/masterlists/masterlist_${i}_structure.txt`);
-}
-
-console.log(`Extracted ${masterlists.length} masterlist structures.`);
-
-for (let i = 0; i < masterlists.length; i++) {
-  const asn1Output = fs.readFileSync(`outputs/masterlists/masterlist_${i}_structure.txt`, 'utf8');
-
-  // Extract the first hex dump using a regex
-  const hexDumpMatch = asn1Output.match(/\[HEX DUMP\]:([A-Fa-f0-9]+)/);
-  if (!hexDumpMatch) {
-    console.error('No hex dump found');
-    process.exit(1);
-  }
-  const hexDump = hexDumpMatch[1];
-
-  // Convert hex dump to binary
-  const binaryDump = Buffer.from(hexDump, 'hex');
-  fs.writeFileSync(`outputs/masterlists/masterlist_${i}_binary_dump.bin`, binaryDump);
-
-  // Parse binary data using OpenSSL and extract individual certificates
-  const asn1ParseOutput = execSync(`openssl asn1parse -inform DER -in outputs/masterlists/masterlist_${i}_binary_dump.bin`, { maxBuffer: 10485770 }).toString();
-  fs.writeFileSync(`outputs/masterlists/masterlist_${i}_asn1_parse_output.txt`, asn1ParseOutput);
-
-  const certificateMatches = asn1ParseOutput.matchAll(/(\d+):d=2\s+hl=4\s+l=\s*(\d+)\s+cons:\s+SEQUENCE/g);
-
-  if (!fs.existsSync(`outputs/cscas/`)) {
-    fs.mkdirSync(`outputs/cscas/`);
+  while ((match = regex.exec(fileContent)) !== null) {
+    const masterlist = match[1].replace(/\s+/g, "");
+    masterlists.push(masterlist);
   }
 
-  if (!fs.existsSync(`outputs/cscas/masterlist_${i}`)) {
-    fs.mkdirSync(`outputs/cscas/masterlist_${i}`);
+  if (!fs.existsSync(masterlist_path)) {
+    fs.mkdirSync(masterlist_path);
   }
 
-  let count = 0;
-  for (const match of certificateMatches) {
-    const startOffset = parseInt(match[1]);
-    const certificateOutput = execSync(`openssl asn1parse -inform DER -in outputs/masterlists/masterlist_${i}_binary_dump.bin -strparse ${startOffset} -out outputs/cscas/masterlist_${i}/cert_${count}.pem`).toString();
-    console.log(`Extracted certificate ${count} to cert_${count}.pem`);
-    count++;
+  for (let i = 0; i < masterlists.length; i++) {
+    fs.writeFileSync(
+      path.join(masterlist_path, `masterlist_${i}.pem`),
+      `-----BEGIN CERTIFICATE-----\n${masterlists[i]}\n-----END CERTIFICATE-----\n`
+    );
   }
-}
 
-console.log('Deduplicating certificates...');
+  console.log(`Extracted ${masterlists.length} masterlists.`);
 
-// Deduplicate certificates
-const uniqueCertificates = new Set<string>();
+  for (let i = 0; i < masterlists.length; i++) {
+    execSync(`openssl asn1parse -in ${path.join(masterlist_path, `masterlist_${i}.pem`)} -inform PEM -i > ${path.join(masterlist_path, `masterlist_${i}_structure.txt`)}`);
+  }
 
-const masterlistDirectories = fs.readdirSync('outputs/cscas/');
+  console.log(`Extracted ${masterlists.length} masterlist structures.`);
 
-masterlistDirectories.forEach((directory) => {
-  const files = fs.readdirSync(`outputs/cscas/${directory}`);
+  for (let i = 0; i < masterlists.length; i++) {
+    const asn1Output = fs.readFileSync(path.join(masterlist_path, `masterlist_${i}_structure.txt`), 'utf8');
 
-  files.forEach((file) => {
-    const filePath = path.join(`outputs/cscas/${directory}`, file);
-    const certContent = fs.readFileSync(filePath);  // Read as binary
-
-    const certBase64 = certContent.toString('base64');  // Convert to base64 for comparison
-
-    if (!uniqueCertificates.has(certBase64)) {
-      uniqueCertificates.add(certBase64);
+    const hexDumpMatch = asn1Output.match(/\[HEX DUMP\]:([A-Fa-f0-9]+)/);
+    if (!hexDumpMatch) {
+      console.error('No hex dump found');
+      process.exit(1);
     }
+    const hexDump = hexDumpMatch[1];
+
+    const binaryDump = Buffer.from(hexDump, 'hex');
+    fs.writeFileSync(path.join(masterlist_path, `masterlist_${i}_binary_dump.bin`), binaryDump);
+
+    const asn1ParseOutput = execSync(`openssl asn1parse -inform DER -in ${path.join(masterlist_path, `masterlist_${i}_binary_dump.bin`)}`, { maxBuffer: 10485770 }).toString();
+    fs.writeFileSync(path.join(masterlist_path, `masterlist_${i}_asn1_parse_output.txt`), asn1ParseOutput);
+
+    const certificateMatches = asn1ParseOutput.matchAll(/(\d+):d=2\s+hl=4\s+l=\s*(\d+)\s+cons:\s+SEQUENCE/g);
+
+    if (!fs.existsSync(csca_path)) {
+      fs.mkdirSync(csca_path);
+    }
+
+    const masterlist_csca_path = path.join(csca_path, `masterlist_${i}`);
+    if (!fs.existsSync(masterlist_csca_path)) {
+      fs.mkdirSync(masterlist_csca_path);
+    }
+
+    let count = 0;
+    for (const match of certificateMatches) {
+      const startOffset = parseInt(match[1]);
+      const certificateOutput = execSync(`openssl asn1parse -inform DER -in ${path.join(masterlist_path, `masterlist_${i}_binary_dump.bin`)} -strparse ${startOffset} -out ${path.join(masterlist_csca_path, `cert_${count}.pem`)}`).toString();
+      console.log(`Extracted certificate ${count} to cert_${count}.pem`);
+      count++;
+    }
+  }
+
+  console.log('Deduplicating certificates...');
+
+  const uniqueCertificates = new Set<string>();
+
+  const masterlistDirectories = fs.readdirSync(csca_path);
+
+  masterlistDirectories.forEach((directory) => {
+    const files = fs.readdirSync(path.join(csca_path, directory));
+
+    files.forEach((file) => {
+      const filePath = path.join(csca_path, directory, file);
+      const certContent = fs.readFileSync(filePath);
+
+      const certBase64 = certContent.toString('base64');
+
+      if (!uniqueCertificates.has(certBase64)) {
+        uniqueCertificates.add(certBase64);
+      }
+    });
   });
-});
 
-// Write unique certificates to new files
-const uniqueCertsDir = 'outputs/unique_pem_masterlist/';
-if (!fs.existsSync(uniqueCertsDir)) {
-  fs.mkdirSync(uniqueCertsDir);
-}
+  const uniqueCertsDir = path.join(csca_path, 'pem_masterlist');
+  if (!fs.existsSync(uniqueCertsDir)) {
+    fs.mkdirSync(uniqueCertsDir);
+  }
 
-let uniqueCertCount = 0;
-uniqueCertificates.forEach((certBase64) => {
-  // const certBuffer = Buffer.from(certBase64, 'base64');  // Convert back to binary
-  const pemCert = `-----BEGIN CERTIFICATE-----\n${certBase64}\n-----END CERTIFICATE-----\n`;
-  fs.writeFileSync(path.join(uniqueCertsDir, `unique_cert_${uniqueCertCount}.pem`), pemCert);
-  uniqueCertCount++;
-});
+  let uniqueCertCount = 0;
+  uniqueCertificates.forEach((certBase64) => {
+    const pemCert = `-----BEGIN CERTIFICATE-----\n${certBase64}\n-----END CERTIFICATE-----\n`;
+    fs.writeFileSync(path.join(uniqueCertsDir, `unique_cert_${uniqueCertCount}.pem`), pemCert);
+    uniqueCertCount++;
+  });
 
-console.log(`Deduplicated and saved ${uniqueCertCount} unique certificates.`);
+  console.log(`Deduplicated and saved ${uniqueCertCount} unique certificates.`);
+}

registry/src/dsc/extract_masterlist_dsc.ts

@@ -1,27 +1,32 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-// Extract pem certificates from ldif file
-const fileContent = fs.readFileSync("inputs/icao_download_section/icaopkd-001-complete-007117.ldif", "utf-8");
-const regex = /userCertificate;binary::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
-let match: RegExpExecArray | null;
+export async function extractMasterlistDsc() {
+  // Extract pem certificates from ldif file
+  const ldif_path = path.join(__dirname, '..', '..', 'inputs', 'icao_download_section', 'icaopkd-001-complete-007117.ldif');
+  const pem_path = path.join(__dirname, '..', '..', 'outputs', 'dsc', 'pem_masterlist');
 
-const certificates: string[] = [];
+  const fileContent = fs.readFileSync(ldif_path, "utf-8");
+  const regex = /userCertificate;binary::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
+  let match: RegExpExecArray | null;
 
-while ((match = regex.exec(fileContent)) !== null) {
-  const certificate = match[1].replace(/\s+/g, "");
-  certificates.push(certificate);
-}
+  const certificates: string[] = [];
 
-if (!fs.existsSync("outputs/dsc/pem_masterlist")) {
-  fs.mkdirSync("outputs/dsc/pem_masterlist");
-}
+  while ((match = regex.exec(fileContent)) !== null) {
+    const certificate = match[1].replace(/\s+/g, "");
+    certificates.push(certificate);
+  }
 
-for (let i = 0; i < certificates.length; i++) {
-  fs.writeFileSync(
-    path.join("outputs/dsc/pem_masterlist/", `certificate_${i}.pem`),
-    `-----BEGIN CERTIFICATE-----\n${certificates[i]}\n-----END CERTIFICATE-----\n`
-  );
-}
+  if (!fs.existsSync(pem_path)) {
+    fs.mkdirSync(pem_path);
+  }
 
-console.log(`Extracted ${certificates.length} certificates.`);
+  for (let i = 0; i < certificates.length; i++) {
+    fs.writeFileSync(
+      path.join(pem_path, `certificate_${i}.pem`),
+      `-----BEGIN CERTIFICATE-----\n${certificates[i]}\n-----END CERTIFICATE-----\n`
+    );
+  }
+
+  console.log(`Extracted ${certificates.length} certificates.`);
+}

registry/src/extractMasterList.ts

@@ -0,0 +1,25 @@
+import { extractMasterlistCsca } from './csca/extract_masterlist_csca';
+import { extractMasterlistDsc } from './dsc/extract_masterlist_dsc';
+
+async function extractMasterList(type: string) {
+    switch (type) {
+        case 'dsc':
+            await extractMasterlistDsc();
+            break;
+        case 'csca':
+            await extractMasterlistCsca();
+            break;
+        case 'all':
+            await extractMasterlistCsca();
+            await extractMasterlistDsc();
+            break;
+        default:
+            console.error('Invalid parameter. Use "dsc", "csca", or "all".');
+    }
+}
+
+// Get the command line argument
+const arg = process.argv[2];
+
+// Call the function with the provided argument
+extractMasterList(arg);