59 lines
1.7 KiB
TypeScript
59 lines
1.7 KiB
TypeScript
import { parse } from 'node-html-parser';
|
|
|
|
const dataFile = Bun.file('../src/data/data.csv');
|
|
const dataWriter = dataFile.writer({ highWaterMark: 1024 * 1024 });
|
|
|
|
const doc = await fetch(
|
|
'https://wiki.bre-forum.de/index.php?title=Bremer_Nahverkehrs-Net(z)_-_Fahrzeugliste'
|
|
)
|
|
.then((r) => r.text())
|
|
.then((html) => parse(html));
|
|
|
|
const tables = doc.querySelectorAll('table.sortable');
|
|
|
|
console.log(tables.length);
|
|
let a = 0;
|
|
tables.forEach(async (t) => {
|
|
if (
|
|
t.firstElementChild?.firstElementChild?.firstElementChild?.tagName !== 'TH' ||
|
|
t.firstElementChild?.firstElementChild.firstElementChild.text !== 'Fahrzeug-\nNr.'
|
|
) {
|
|
return;
|
|
}
|
|
const columns = t.firstElementChild.firstElementChild
|
|
.getElementsByTagName('th')
|
|
.map((e) => e.text.replaceAll(/-?\n/g, ''));
|
|
|
|
// dataWriter.write(columns.join(';'));
|
|
// dataWriter.write('\n');
|
|
const rows = t.firstElementChild.querySelectorAll('tr:has(> td)').map((r) => {
|
|
const cellElements = r.getElementsByTagName('td');
|
|
cellElements.splice(1, 1);
|
|
const cells = cellElements.map((c, i) =>
|
|
(i == 2 ? c.innerHTML : c.innerText).replaceAll('<br>', ',').replaceAll(/-?\n/g, '').trim()
|
|
);
|
|
const imgLink = cellElements[1]?.querySelector('a[href]');
|
|
if (imgLink !== null) {
|
|
cells[1] = imgLink!.attributes.href.split('Datei:')[1];
|
|
}
|
|
return Number(cells[0]) >= 3000 && Number(cells[0]) < 3500 ? cells.join(';') : '';
|
|
});
|
|
|
|
rows.forEach((r) => {
|
|
if (r.length <= 0) {
|
|
return;
|
|
}
|
|
dataWriter.write(r);
|
|
dataWriter.write('\n');
|
|
});
|
|
|
|
console.log(columns, rows.length);
|
|
|
|
await dataWriter.flush();
|
|
a++;
|
|
// console.log(t.firstElementChild?.firstElementChild.textContent);
|
|
});
|
|
console.log(a);
|
|
|
|
dataWriter.end();
|