anquet-migrator/update.js

import { Directus } from '@directus/sdk';
import TurndownService from 'turndown';
import fs from 'fs';
import he from 'he';
import * as cheerio from 'cheerio';
import slug from 'slug';
import Redis from 'ioredis';
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import

dotenv.config()
const directus = new Directus(process.env.DIRECTUS_HOST);
const turndownService = new TurndownService()
const htmlString = fs.readFileSync('Solutions.xml').toString();
const $ = cheerio.load(htmlString);
const redis = new Redis();

async function start() {
  await directus.auth.login({ email: process.env.DIRECTUS_EMAIL, password: process.env.DIRECTUS_PASSWORD })
    .catch(() => {
      console.error('Invalid credentials');
    });

  $('solution-article').each(async function(i, element) {
    let parentTitle = $("name", $(this).parent().parent()).html();
    let title = he.decode($(this).children("title").html());
    let description = he.decode($(this).children("description").html());
    let images = $(description).children("img");
    const title_slug = slug(parentTitle)+"/"+slug(title)
    let original_id = parseInt(he.decode($(this).children("id").html()));

    await redis.hset("pageurls", original_id, title_slug);
    console.log('Parsing images');
    images.each(async function(e, image) {
      let imageUrl = $(image).attr('src');
      let file = directus.files.import({
        url: imageUrl,
      });
      await redis.hset("images", imageUrl, 'https://directus.anquet.com/assets/' + file.id + '-' + file.width + '-' + file.height);
    });
    description = turndownService.turndown(description);

    console.log('Saving page '+title);
    await redis.hset("pages", original_id, JSON.stringify({
      title: title,
      content: description,
      status: 'published',
      original_id: original_id,
      url: slug(title),
    }))
    return Promise.resolve();
  });

  const articles = directus.items('omn2omn3');
  const translations = directus.items('omn2omn3_translations');
  const pages = await redis.hgetall("pages");
  const images = await redis.hgetall("images");
  Object.keys(pages).forEach(async function (original_id) { 
    const page = JSON.parse(pages[original_id])
    const reg = new RegExp(/https:\/\/support.anquet.com\/support\/solutions\/articles\/(\d+)-?/gm);
    for (const match of page.content.matchAll(reg)) {
      const title_slug = await redis.hget("pageurls", match[1]);
      if (title_slug) {
        const reg2 = new RegExp("/\(https:\/\/support.anquet.com\/support\/solutions\/articles\/"+match[1]+"-?.*\)/gm");
        page.content.replace(reg2, '('+title_slug+')');
      }
    }
    Object.keys(images).forEach(async function (image_url) { 
      page.content.replace(image_url, images[image_url])
    });
    const content = await translations.createOne({
      languages_code: 'en-GB',
      markdown: page.content,
    });

    await articles.createOne({
      title: page.title,
      translations: [content.id],
      status: 'published',
      original_id: original_id,
      url: page.url,
    }).then(function() {
      console.log("Added "+page.title);
    });

    return Promise.resolve();
  })
}

start();