anquet-migrator/update.js

import { Directus } from '@directus/sdk';
import TurndownService from 'turndown';
import fs from 'fs';
import he from 'he';
import * as cheerio from 'cheerio';
import slug from 'slug';
import Redis from 'ioredis';
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import

dotenv.config()
const directus = new Directus(process.env.DIRECTUS_HOST);
const turndownService = new TurndownService()
const htmlString = fs.readFileSync('Solutions.xml').toString();
const $ = cheerio.load(htmlString);
const redis = new Redis();
await directus.auth.login({ email: process.env.DIRECTUS_EMAIL, password: process.env.DIRECTUS_PASSWORD })
  .catch(() => {
    console.error('Invalid credentials');
  });

async function start() {
  const a = $('solution-article').toArray();
  for (let i = 0; i < a.length; i++) {
    const article = a[i]
    let parentTitle = $("name", $(article).parent().parent()).html();
    let title = he.decode($(article).children("title").html());
    let description = he.decode($(article).children("description").html());
    let images = $(description).children("img").toArray();
    for (let j = 0; j < images.length; j++) {
      let imageUrl = $(images[j]).attr('src');
      const n = await redis.hget("images", imageUrl);
      if (!n) {
        await redis.hset("images", imageUrl, "");
      }
    }
    const title_slug = slug(parentTitle)+"/"+slug(title)
    let original_id = parseInt(he.decode($(article).children("id").html()));

    await redis.hset("pageurls", original_id, title_slug);
    description = turndownService.turndown(description);

    console.log('Saving page '+title);
    await redis.hset("pages", original_id, JSON.stringify({
      title: title,
      content: description,
      status: 'published',
      original_id: original_id,
      url: slug(title),
    }))
  }

  const t = await redis.hgetall("images");
  const toImport = Object.keys(t)

  for (let i = 0; i < toImport.length; i++) {
    try {
      const n = await redis.hget("images", toImport[i]);
      if (n === "") {
        console.log("Saving image "+toImport[i])
        let file = await directus.files.import({
          url: toImport[i],
        });
        await redis.hset("images", toImport[i], 'https://directus.anquet.com/assets/' + file.id + '-' + file.width + '-' + file.height);
      }
    } catch (e) {
      console.error(e.getMessage())
      // Can't import this image.
    }
  }

  const pages = await redis.hgetall("pages");
  const p = Object.keys(pages);
  for (let i = 0; i < p.length; i++) {
    const original_id = p[i];
    const page = JSON.parse(pages[original_id])
    const reg = new RegExp(/https:\/\/support.anquet.com\/support\/solutions\/articles\/(\d+)-?/gm);
    for (const match of page.content.matchAll(reg)) {
      const title_slug = await redis.hget("pageurls", match[1]);
      if (title_slug) {
        const reg2 = new RegExp("/\(https:\/\/support.anquet.com\/support\/solutions\/articles\/"+match[1]+"-?.*\)/gm");
        page.content = page.content.replace(reg2, '('+title_slug+')');
      }
    }

    for (let i = 0; i < toImport.length; i++) {
      const image_url = toImport[i]
      if (page.content.match(image_url)) {
        const newurl = await redis.hget("images", image_url);
        if (newurl) {
          page.content = page.content.replace(image_url, newurl)
        }
      }
    }

    await redis.hset("pages", original_id, JSON.stringify(page))
  }
}

async function end() {
  const articles = directus.items('omn2omn3');
  const translations = directus.items('omn2omn3_translations');
  const pages = await redis.hgetall("pages");
  const p = Object.keys(pages);
  for (let i = 0; i < p.length; i++) {
    const original_id = p[i];
    const page = JSON.parse(pages[original_id])
    const content = await translations.createOne({
      languages_code: 'en-GB',
      markdown: page.content,
    });

    await articles.createOne({
      title: page.title,
      translations: [content.id],
      status: 'published',
      original_id: original_id,
      url: page.url,
    }).then(function() {
      console.log("Added "+page.title);
    });
  }
}

await start();
await end();