import { Directus } from '@directus/sdk'; import TurndownService from 'turndown'; import fs from 'fs'; import he from 'he'; import * as cheerio from 'cheerio'; import slug from 'slug'; import Redis from 'ioredis'; import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import dotenv.config() const directus = new Directus(process.env.DIRECTUS_HOST); const turndownService = new TurndownService() const htmlString = fs.readFileSync('Solutions.xml').toString(); const $ = cheerio.load(htmlString); const redis = new Redis(); await directus.auth.login({ email: process.env.DIRECTUS_EMAIL, password: process.env.DIRECTUS_PASSWORD }) .catch(() => { console.error('Invalid credentials'); }); async function start() { const a = $('solution-article').toArray(); for (let i = 0; i < a.length; i++) { const article = a[i] let parentTitle = $("name", $(article).parent().parent()).html(); let title = he.decode($(article).children("title").html()); let description = he.decode($(article).children("description").html()); let images = $(description).children("img").toArray(); for (let j = 0; j < images.length; j++) { let imageUrl = $(images[j]).attr('src'); const n = await redis.hget("images", imageUrl); if (!n) { await redis.hset("images", imageUrl, ""); } } const title_slug = slug(parentTitle)+"/"+slug(title) let original_id = parseInt(he.decode($(article).children("id").html())); await redis.hset("pageurls", original_id, title_slug); description = turndownService.turndown(description); console.log('Saving page '+title); await redis.hset("pages", original_id, JSON.stringify({ title: title, content: description, status: 'published', original_id: original_id, url: slug(title), })) } const t = await redis.hgetall("images"); const toImport = Object.keys(t) for (let i = 0; i < toImport.length; i++) { try { const n = await redis.hget("images", toImport[i]); if (n === "") { console.log("Saving image "+toImport[i]) let file = await directus.files.import({ url: toImport[i], }); await redis.hset("images", toImport[i], 'https://directus.anquet.com/assets/' + file.id + '-' + file.width + '-' + file.height); } } catch (e) { console.error(e.getMessage()) // Can't import this image. } } const pages = await redis.hgetall("pages"); const p = Object.keys(pages); for (let i = 0; i < p.length; i++) { const original_id = p[i]; const page = JSON.parse(pages[original_id]) const reg = new RegExp(/https:\/\/support.anquet.com\/support\/solutions\/articles\/(\d+)-?/gm); for (const match of page.content.matchAll(reg)) { const title_slug = await redis.hget("pageurls", match[1]); if (title_slug) { const reg2 = new RegExp("/\(https:\/\/support.anquet.com\/support\/solutions\/articles\/"+match[1]+"-?.*\)/gm"); page.content = page.content.replace(reg2, '('+title_slug+')'); } } for (let i = 0; i < toImport.length; i++) { const image_url = toImport[i] if (page.content.match(image_url)) { const newurl = await redis.hget("images", image_url); if (newurl) { page.content = page.content.replace(image_url, newurl) } } } await redis.hset("pages", original_id, JSON.stringify(page)) } } async function end() { const articles = directus.items('omn2omn3'); const translations = directus.items('omn2omn3_translations'); const pages = await redis.hgetall("pages"); const p = Object.keys(pages); for (let i = 0; i < p.length; i++) { const original_id = p[i]; const page = JSON.parse(pages[original_id]) const content = await translations.createOne({ languages_code: 'en-GB', markdown: page.content, }); await articles.createOne({ title: page.title, translations: [content.id], status: 'published', original_id: original_id, url: page.url, }).then(function() { console.log("Added "+page.title); }); } } await start(); await end();