mirror of
https://github.com/selfxyz/self.git
synced 2026-04-27 03:01:15 -04:00
added masterlists parsing and csca deduplication
This commit is contained in:
0xturboblitz
4
registry/.gitignore
vendored
4
registry/.gitignore
vendored
@@ -1,3 +1,5 @@
|
||||
node_modules/
|
||||
certificates
|
||||
csca_certificates
|
||||
cscas
|
||||
masterlists
|
||||
unique_cscas
|
||||
@@ -39,14 +39,14 @@ ts-node src/dsc/extract_sig_algs.ts
|
||||
|
||||
### CSCAs (WIP)
|
||||
|
||||
Extract pem certificates from ldif file:
|
||||
Extract pem certificates from all the masterlists from the ldif file:
|
||||
```
|
||||
ts-node src/csca/extract_certificates.ts
|
||||
ts-node src/csca/extract_masterlists.ts
|
||||
```
|
||||
|
||||
Extract readable public keys from pem certicates:
|
||||
Visualize the content of a PEM file:
|
||||
```
|
||||
ts-node src/csca/extract_pubkeys.ts
|
||||
openssl x509 -text -in outputs/unique_cscas/unique_cert_0.pem
|
||||
```
|
||||
|
||||
More info: [ICAO website](https://www.icao.int/Security/FAL/PKD/Pages/icao-master-list.aspx)
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// extract certificates from ldif file
|
||||
const fileContent = fs.readFileSync("inputs/icao_download_section/icaopkd-002-complete-000243.ldif", "utf-8");
|
||||
const regex = /pkdMasterListContent::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
const certificates: string[] = [];
|
||||
|
||||
while ((match = regex.exec(fileContent)) !== null) {
|
||||
const certificate = match[1].replace(/\s+/g, "");
|
||||
certificates.push(certificate);
|
||||
}
|
||||
|
||||
if (!fs.existsSync("outputs/csca_certificates/")) {
|
||||
fs.mkdirSync("outputs/csca_certificates/");
|
||||
}
|
||||
|
||||
for (let i = 0; i < certificates.length; i++) {
|
||||
fs.writeFileSync(
|
||||
path.join("outputs/csca_certificates/", `certificate_${i}.pem`),
|
||||
`-----BEGIN CERTIFICATE-----\n${certificates[i]}\n-----END CERTIFICATE-----\n`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Extracted ${certificates.length} certificates.`);
|
||||
111
registry/src/csca/extract_masterlists.ts
Normal file
111
registry/src/csca/extract_masterlists.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as util from 'util';
|
||||
import { execSync } from 'child_process';
|
||||
const execAsync = util.promisify(execSync);
|
||||
|
||||
// extract masterlists from ICAO ldif file
|
||||
const fileContent = fs.readFileSync("inputs/icao_download_section/icaopkd-002-complete-000243.ldif", "utf-8");
|
||||
const regex = /pkdMasterListContent::\s*([\s\S]*?)(?=\w+:|\n\n|$)/g;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
const masterlists: string[] = [];
|
||||
|
||||
while ((match = regex.exec(fileContent)) !== null) {
|
||||
const masterlist = match[1].replace(/\s+/g, "");
|
||||
masterlists.push(masterlist);
|
||||
}
|
||||
|
||||
if (!fs.existsSync("outputs/masterlists/")) {
|
||||
fs.mkdirSync("outputs/masterlists/");
|
||||
}
|
||||
|
||||
for (let i = 0; i < masterlists.length; i++) {
|
||||
fs.writeFileSync(
|
||||
path.join("outputs/masterlists/", `masterlist_${i}.pem`),
|
||||
`-----BEGIN CERTIFICATE-----\n${masterlists[i]}\n-----END CERTIFICATE-----\n`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Extracted ${masterlists.length} masterlists.`);
|
||||
|
||||
for (let i = 0; i < masterlists.length; i++) {
|
||||
execSync(`openssl asn1parse -in outputs/masterlists/masterlist_${i}.pem -inform PEM -i > outputs/masterlists/masterlist_${i}_structure.txt`);
|
||||
}
|
||||
|
||||
console.log(`Extracted ${masterlists.length} masterlist structures.`);
|
||||
|
||||
for (let i = 0; i < masterlists.length; i++) {
|
||||
const asn1Output = fs.readFileSync(`outputs/masterlists/masterlist_${i}_structure.txt`, 'utf8');
|
||||
|
||||
// Extract the first hex dump using a regex
|
||||
const hexDumpMatch = asn1Output.match(/\[HEX DUMP\]:([A-Fa-f0-9]+)/);
|
||||
if (!hexDumpMatch) {
|
||||
console.error('No hex dump found');
|
||||
process.exit(1);
|
||||
}
|
||||
const hexDump = hexDumpMatch[1];
|
||||
|
||||
// Convert hex dump to binary
|
||||
const binaryDump = Buffer.from(hexDump, 'hex');
|
||||
fs.writeFileSync(`outputs/masterlists/masterlist_${i}_binary_dump.bin`, binaryDump);
|
||||
|
||||
// Parse binary data using OpenSSL and extract individual certificates
|
||||
const asn1ParseOutput = execSync(`openssl asn1parse -inform DER -in outputs/masterlists/masterlist_${i}_binary_dump.bin`, { maxBuffer: 10485770 }).toString();
|
||||
fs.writeFileSync(`outputs/masterlists/masterlist_${i}_asn1_parse_output.txt`, asn1ParseOutput);
|
||||
|
||||
const certificateMatches = asn1ParseOutput.matchAll(/(\d+):d=2\s+hl=4\s+l=\s*(\d+)\s+cons:\s+SEQUENCE/g);
|
||||
|
||||
if (!fs.existsSync(`outputs/cscas/`)) {
|
||||
fs.mkdirSync(`outputs/cscas/`);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(`outputs/cscas/masterlist_${i}`)) {
|
||||
fs.mkdirSync(`outputs/cscas/masterlist_${i}`);
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const match of certificateMatches) {
|
||||
const startOffset = parseInt(match[1]);
|
||||
const certificateOutput = execSync(`openssl asn1parse -inform DER -in outputs/masterlists/masterlist_${i}_binary_dump.bin -strparse ${startOffset} -out outputs/cscas/masterlist_${i}/cert_${count}.pem`).toString();
|
||||
console.log(`Extracted certificate ${count} to cert_${count}.pem`);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Deduplicating certificates...');
|
||||
|
||||
// Deduplicate certificates
|
||||
const uniqueCertificates = new Set<string>();
|
||||
|
||||
const masterlistDirectories = fs.readdirSync('outputs/cscas/');
|
||||
|
||||
masterlistDirectories.forEach((directory) => {
|
||||
const files = fs.readdirSync(`outputs/cscas/${directory}`);
|
||||
|
||||
files.forEach((file) => {
|
||||
const filePath = path.join(`outputs/cscas/${directory}`, file);
|
||||
const certContent = fs.readFileSync(filePath); // Read as binary
|
||||
|
||||
const certBase64 = certContent.toString('base64'); // Convert to base64 for comparison
|
||||
|
||||
if (!uniqueCertificates.has(certBase64)) {
|
||||
uniqueCertificates.add(certBase64);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Write unique certificates to new files
|
||||
const uniqueCertsDir = 'outputs/unique_cscas/';
|
||||
if (!fs.existsSync(uniqueCertsDir)) {
|
||||
fs.mkdirSync(uniqueCertsDir);
|
||||
}
|
||||
|
||||
let uniqueCertCount = 0;
|
||||
uniqueCertificates.forEach((certBase64) => {
|
||||
const certBuffer = Buffer.from(certBase64, 'base64'); // Convert back to binary
|
||||
fs.writeFileSync(path.join(uniqueCertsDir, `unique_cert_${uniqueCertCount}.pem`), certBuffer);
|
||||
uniqueCertCount++;
|
||||
});
|
||||
|
||||
console.log(`Deduplicated and saved ${uniqueCertCount} unique certificates.`);
|
||||
@@ -1,154 +0,0 @@
|
||||
import * as fs from 'fs';
|
||||
import * as util from 'util';
|
||||
import { exec } from 'child_process';
|
||||
const execAsync = util.promisify(exec);
|
||||
|
||||
// Count the number of files in certificates/
|
||||
const numCertificates = fs.readdirSync('outputs/csca_certificates/').length;
|
||||
const concurrencyLimit = 1; // Number of tasks to run at once
|
||||
|
||||
const publicKeysParsed: {
|
||||
signatureAlgorithm: string,
|
||||
modulus?: string,
|
||||
exponent?: string,
|
||||
publicKeyAlgorithm?: string
|
||||
publicKeyBit?: string
|
||||
pub?: string
|
||||
fieldType?: string
|
||||
prime?: string
|
||||
a?: string
|
||||
b?: string
|
||||
generator?: string
|
||||
order?: string
|
||||
cofactor?: string
|
||||
}[] = [];
|
||||
|
||||
async function main() {
|
||||
for (let i = 0; i < numCertificates; i += concurrencyLimit) {
|
||||
const tasks: any = [];
|
||||
for (let j = 0; j < concurrencyLimit && i + j < numCertificates; j++) {
|
||||
tasks.push(extractModulus(i + j));
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
|
||||
// console.log('publicKeysParsed 0', publicKeysParsed[0]);
|
||||
// const filteredPublicKeysParsed = publicKeysParsed.filter(item => item !== null);
|
||||
// fs.writeFileSync('public_keys_parsed.json', JSON.stringify(filteredPublicKeysParsed, null, 2));
|
||||
// console.log("public_keys_parsed.json written!")
|
||||
}
|
||||
|
||||
async function extractModulus(i: number): Promise<void> {
|
||||
try {
|
||||
const certTextres = await execAsync(`openssl x509 -text -in outputs/certificates/certificate_${i}.pem`);
|
||||
const certText = certTextres.stdout as string;
|
||||
// const signatureAlgorithm = (certText.match(/Signature Algorithm: (.*)/) as RegExpExecArray)[1].trim();
|
||||
|
||||
console.log('certText', certText)
|
||||
|
||||
// const issuerRegex = /Issuer: ([^\n]+)/;
|
||||
// const issuer = extractData(issuerRegex, certText);
|
||||
|
||||
// // console.log('issuer', issuer)
|
||||
|
||||
// const pubkey = parsePubkey(certText, signatureAlgorithm);
|
||||
|
||||
// if (!pubkey) {
|
||||
// console.error(`Failed to extract data from certificate ${i}`);
|
||||
// return;
|
||||
// }
|
||||
|
||||
// publicKeysParsed[i] = {
|
||||
// signatureAlgorithm,
|
||||
// issuer,
|
||||
// ...pubkey,
|
||||
// }
|
||||
} catch (error) {
|
||||
console.error(`Failed to extract data from certificate ${i}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parsePubkey(certText: string, signatureAlgorithm: string): any {
|
||||
if (
|
||||
signatureAlgorithm.includes("sha256WithRSAEncryption")
|
||||
|| signatureAlgorithm.includes("rsassaPss")
|
||||
|| signatureAlgorithm.includes("sha1WithRSAEncryption")
|
||||
|| signatureAlgorithm.includes("sha512WithRSAEncryption")
|
||||
) {
|
||||
const modulusRegex = /Modulus:\s+([0-9a-f:\s]+?)\s+Exponent:/;
|
||||
const exponentRegex = /Exponent:\s+(\d+)/;
|
||||
|
||||
const modulusMatch = certText.match(modulusRegex);
|
||||
const exponentMatch = certText.match(exponentRegex);
|
||||
|
||||
const modulusHex = modulusMatch ? modulusMatch[1].replace(/[\s:]/g, '') : '';
|
||||
const exponent = exponentMatch ? exponentMatch[1] : '';
|
||||
|
||||
if (!modulusHex) {
|
||||
console.error(`Modulus not found`);
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
modulus: BigInt('0x' + modulusHex).toString(),
|
||||
exponent: exponent
|
||||
};
|
||||
} else if (
|
||||
signatureAlgorithm.includes("ecdsa-with-SHA1")
|
||||
|| signatureAlgorithm.includes("ecdsa-with-SHA384")
|
||||
|| signatureAlgorithm.includes("ecdsa-with-SHA256")
|
||||
|| signatureAlgorithm.includes("ecdsa-with-SHA512")
|
||||
) {
|
||||
|
||||
const publicKeyAlgorithmRegex = /Public Key Algorithm: ([^\n]+)/;
|
||||
const publicKeyBitRegex = /Public-Key: \((\d+) bit\)/;
|
||||
const pubRegex = /pub:\n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const fieldTypeRegex = /Field Type: ([^\n]+)/;
|
||||
const primeRegex = /Prime:\n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const aRegex = /A:\s+\n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const bRegex = /B:\s+\n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const generatorRegex = /Generator \(uncompressed\):\n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const orderRegex = /Order: \n([0-9A-Fa-f:\n ]+?)\n\s{4}/;
|
||||
const cofactorRegex = /Cofactor:\s+(\d+)/;
|
||||
|
||||
// Extracting fields
|
||||
const publicKeyAlgorithm = extractData(publicKeyAlgorithmRegex, certText);
|
||||
const publicKeyBit = extractData(publicKeyBitRegex, certText);
|
||||
const pub = extractData(pubRegex, certText);
|
||||
const fieldType = extractData(fieldTypeRegex, certText);
|
||||
const prime = extractData(primeRegex, certText);
|
||||
const a = extractData(aRegex, certText);
|
||||
const b = extractData(bRegex, certText);
|
||||
const generator = extractData(generatorRegex, certText);
|
||||
const order = extractData(orderRegex, certText);
|
||||
const cofactor = extractData(cofactorRegex, certText);
|
||||
|
||||
if (!prime) {
|
||||
console.error(`Prime not found`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
publicKeyAlgorithm: publicKeyAlgorithm,
|
||||
publicKeyBit: publicKeyBit,
|
||||
pub: hexToDecimal(pub as string),
|
||||
fieldType: fieldType,
|
||||
prime: hexToDecimal(prime as string),
|
||||
a: hexToDecimal(a as string),
|
||||
b: hexToDecimal(b as string),
|
||||
generator: hexToDecimal(generator as string),
|
||||
order: hexToDecimal(order as string),
|
||||
cofactor: cofactor,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function extractData(regex: RegExp, text: string): string | null {
|
||||
const match = text.match(regex);
|
||||
return match ? match[1].trim().replace(/\n/g, '') : null;
|
||||
}
|
||||
|
||||
function hexToDecimal(hexString: string): string {
|
||||
return BigInt("0x" + hexString.replace(/[\n: ]/g, '')).toString();
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user