kolaytercih/scripts/ingest.ts

/**
 * YÖK Atlas verisini SQLite'a yükler.
 *
 * Kaynak: yokatlas-dataset-2025 (github.com/MorphaxTheDeveloper/yokatlas-dataset-2025)
 * — YÖK Atlas'ın halka açık verisinin CSV dökümü. Canlı API'den tazeleme
 * (yokatlas-py'nin kullandığı JSON API) v2'de eklenecek.
 *
 * Kullanım: npx tsx scripts/ingest.ts <tum_bolumler.csv yolu>
 */
import { createReadStream, existsSync, mkdirSync, rmSync } from "node:fs";
import { createInterface } from "node:readline";
import path from "node:path";
import Database from "better-sqlite3";

const YEARS = [2021, 2022, 2023, 2024] as const;

const csvPath = process.argv[2];
if (!csvPath || !existsSync(csvPath)) {
  console.error("Kullanım: npx tsx scripts/ingest.ts <tum_bolumler.csv yolu>");
  process.exit(1);
}

const dbDir = path.join(process.cwd(), "data");
mkdirSync(dbDir, { recursive: true });
const dbPath = path.join(dbDir, "yokatlas.db");
rmSync(dbPath, { force: true });

const db = new Database(dbPath);
db.pragma("journal_mode = WAL");

const yearCols = YEARS.flatMap((y) => [
  `sira${y} INTEGER`,
  `puan${y} REAL`,
  `kontenjan${y} INTEGER`,
  `yerlesen${y} INTEGER`,
]);

db.exec(`
  CREATE TABLE programs (
    id TEXT PRIMARY KEY,
    isim TEXT NOT NULL,
    universite TEXT NOT NULL,
    unitur TEXT,
    il TEXT,
    fakulte TEXT,
    tur TEXT NOT NULL,
    sure INTEGER,
    onlisans INTEGER NOT NULL DEFAULT 0,
    ${yearCols.join(",\n    ")}
  );
  CREATE INDEX idx_programs_tur_sira ON programs (tur, sira2024);
  CREATE INDEX idx_programs_il ON programs (il);
`);

// CSV satırlarını RFC-4180 tırnak kurallarıyla böler (alan içi virgül/tırnak destekli)
function splitCsvLine(line: string): string[] {
  const out: string[] = [];
  let cur = "";
  let inQuotes = false;
  for (let i = 0; i < line.length; i++) {
    const ch = line[i];
    if (inQuotes) {
      if (ch === '"') {
        if (line[i + 1] === '"') {
          cur += '"';
          i++;
        } else {
          inQuotes = false;
        }
      } else {
        cur += ch;
      }
    } else if (ch === '"') {
      inQuotes = true;
    } else if (ch === ",") {
      out.push(cur);
      cur = "";
    } else {
      cur += ch;
    }
  }
  out.push(cur);
  return out;
}

function toInt(v: string | undefined): number | null {
  if (!v) return null;
  const n = Number.parseFloat(v);
  return Number.isFinite(n) ? Math.round(n) : null;
}

function toReal(v: string | undefined): number | null {
  if (!v) return null;
  const n = Number.parseFloat(v);
  return Number.isFinite(n) ? n : null;
}

// Kaynak CSV'de 0, "veri yok" anlamına gelir (sıralama/puan 0 olamaz)
function positiveOrNull(n: number | null): number | null {
  return n != null && n > 0 ? n : null;
}

async function main() {
  const columns = [
    "id",
    "isim",
    "universite",
    "unitur",
    "il",
    "fakulte",
    "tur",
    "sure",
    "onlisans",
    ...YEARS.flatMap((y) => [
      `sira${y}`,
      `puan${y}`,
      `kontenjan${y}`,
      `yerlesen${y}`,
    ]),
  ];
  const insert = db.prepare(
    `INSERT OR REPLACE INTO programs (${columns.join(",")})
     VALUES (${columns.map(() => "?").join(",")})`
  );

  const rl = createInterface({
    input: createReadStream(csvPath, "utf8"),
    crlfDelay: Infinity,
  });

  let header: string[] | null = null;
  let idx: Record<string, number> = {};
  const rows: unknown[][] = [];
  let skipped = 0;

  for await (const line of rl) {
    if (!header) {
      header = splitCsvLine(line);
      idx = Object.fromEntries(header.map((h, i) => [h, i]));
      continue;
    }
    if (!line.trim()) continue;
    const f = splitCsvLine(line);
    const get = (col: string) => f[idx[col]]?.trim() ?? "";

    const id = get("id");
    const isim = get("isim");
    const universite = get("universite");
    const tur = get("tur");
    if (!id || !isim || !universite || !tur) {
      skipped++;
      continue;
    }

    rows.push([
      id,
      isim,
      universite,
      get("unitur") || null,
      get("il") || null,
      get("fakulte") || null,
      tur,
      toInt(get("sure")),
      toInt(get("onlisans")) ?? 0,
      ...YEARS.flatMap((y) => [
        positiveOrNull(toInt(get(`sira${y}`))),
        positiveOrNull(toReal(get(`puan${y}`))),
        toInt(get(`kontenjan${y}`)),
        toInt(get(`yerlesen${y}`)),
      ]),
    ]);
  }

  const insertAll = db.transaction((all: unknown[][]) => {
    for (const r of all) insert.run(...r);
  });
  insertAll(rows);

  const count = db
    .prepare("SELECT COUNT(*) AS c FROM programs")
    .get() as { c: number };
  const withSira = db
    .prepare("SELECT COUNT(*) AS c FROM programs WHERE sira2024 IS NOT NULL")
    .get() as { c: number };
  console.log(`Yüklendi: ${count.c} program (${skipped} satır atlandı)`);
  console.log(`2024 sıralaması olan: ${withSira.c}`);
  console.log(`DB: ${dbPath}`);
}

main().then(() => db.close());