Merge pull request #17 from skohub-io/16-upload-by-url

Add Upload by URL feature

public/index.html

@@ -45,6 +45,8 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
               </label>
               <input type="file" id="file_upload" class="form-control-file" name="uploaded_file" />
               <input type="hidden" name="id" id="id" value="" />
+              <label for="fileUrl">...or provide link to Turtle File (<a href="https://raw.githubusercontent.com/dini-ag-kim/hochschulfaechersystematik/master/hochschulfaechersystematik.ttl">Example</a>)</label>
+              <input type="text" id="fileUrl" name="fileUrl" />
               <input disabled type="submit" value="Upload File to Reconcile Service!" class="btn btn-default" />
             </div>
           </form>
@@ -101,6 +103,7 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
   const account = document.querySelector("#account")
   const language = document.querySelector("#language")
   const file_upload = document.querySelector("#file_upload")
+  const fileUrl = document.querySelector("#fileUrl")
   const id = document.querySelector("#id")
   const submit = document.querySelector("input[type=submit]")
   const spinner = document.querySelector("#spinner")
@@ -115,7 +118,7 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
   id.value = crypto.randomUUID()
 
   function disableSubmit() {
-    if (account.value.length > 0 && language.value.length > 0 && file_upload.value.length > 0) {
+    if (account.value.length > 0 && language.value.length > 0 && (file_upload.value.length > 0 || fileUrl.value.length > 0)) {
       submit.removeAttribute("disabled")
     } else {
       submit.setAttribute("disabled", "disabled")
@@ -148,6 +151,10 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
     disableSubmit()
   })
 
+  fileUrl.addEventListener("input", (e) => {
+    disableSubmit()
+  })
+
   submit.addEventListener("click", (e) => {
     spinner.style.display = "inline-block"
     getData()

src/index.js

@@ -39,10 +39,18 @@ app.post(
   upload.single("uploaded_file"),
   async function(req, res, next) {
     try {
-      const filePath = req.file.path;
+      const filePath = req?.file?.path;
+      const fileUrl = req.body.fileUrl;
+      const account = req.body.account;
       const id = req.body.id;
       const language = req.body.language
-      await publishToReconciliation(filePath, id, language);
+      await publishToReconciliation({
+        filePath,
+        fileUrl,
+        account,
+        id,
+        language
+      });
       res.redirect("/" + "?id=" + id);
     } catch (error) {
       next(error);
@@ -55,8 +63,9 @@ const showError = (error) => (
       <br>
       <details>
       <summary>Error</summary>
+        ${error.message}
         <pre>
-        ${JSON.stringify(error, null, 2)}
+        ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}
         </pre>
       </details>
 `
@@ -77,24 +86,25 @@ app.use((error, req, res, next) => {
     res
       .status(500)
       .send(`
-        Something went wrong while processing the data.Please check the logs. <a href='/'> Go back</a>
+        Something went wrong while processing the data. Please check the logs. <a href='/'> Go back</a>
       ${showError(error)}
     `
       );
   } else if (error.name === "NoPrefNamespaceUriError") {
     res
       .status(400)
       .send(`
-      Please provide a <a href="https://vocab.org/vann/#preferredNamespaceUri"> preferredNamespaceURI</a> 
+      <p>Please provide a <a href="https://vocab.org/vann/#preferredNamespaceUri"> preferredNamespaceURI</a> 
       for your Concept Scheme.
       See <a href="https://github.com/dini-ag-kim/hcrt/blob/84271e3e499c746e211f95297ba451cc547e89d1/hcrt.ttl#L12" > here</a> for an example.
-      < br >
+      </p>
         <a href='/'>Go back</a>
       ${showError(error)}
     `
       );
   }
   else {
+    console.log(error.message)
     res
       .status(500)
       .send(

src/publishToReconciliation/handleData.js

@@ -6,6 +6,7 @@ import { buildJSON } from "./buildJSON.js";
 import { esClient } from "../elastic/connect.js";
 import { writeLog } from "./writeLog.js";
 import { config } from "../config.js";
+import { ReconcileData } from "../types.js";
 
 const esIndex = config.es_index;
 
@@ -21,22 +22,39 @@ function parseFileError(message, error) {
   this.name = "parseFileError";
 }
 
-export const parseFile = async (filePath, log) => {
+export class IllegalCharacterError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "IllegalCharacterError"
+  }
+}
+
+async function getTurtleString(filePath, fileUrl) {
+  if (filePath) {
+    const ttlString = await fs.readFileSync(filePath).toString();
+    return ttlString
+  } else if (fileUrl) {
+    const response = await fetch(fileUrl)
+    const ttlString = await response.text()
+    return ttlString
+  }
+}
+
+/**
+  * @param {ReconcileData} reconcileData
+  * @param {Object} log
+  */
+export const parseFile = async (reconcileData, log) => {
   var data = [];
-  const account = path.basename(path.dirname(filePath));
+  const account = reconcileData.account;
   try {
-    console.log(`> Read and parse ${account}/${path.basename(filePath)} ...`);
     if (!/[a-zA-Z0-9]/.test(account.slice(0, 1))) {
-      console.log(
-        `> Invalid data: account must start with a letter or a number. Instead, its value is: ${account}`
-      );
+      throw new IllegalCharacterError(`Invalid data: account must start with a letter or a number. Instead, its value is: ${account}`)
     }
-    const ttlString = await fs.readFileSync(filePath).toString();
+    const ttlString = await getTurtleString(reconcileData.filePath, reconcileData.fileUrl);
     const j = await buildJSON(ttlString.toString(), account);
     if (!/[a-zA-Z0-9]/.test(j.dataset.slice(0, 1))) {
-      console.log(
-        `> Invalid data: dataset must start with a letter or a number. Instead, its value is: ${j.dataset}`
-      );
+      throw new IllegalCharacterError(`Invalid data: dataset must start with a letter or a number. Instead, its value is: ${j.dataset}`)
     }
     log.status = "processing";
     log.account = account;

src/publishToReconciliation/index.js

@@ -1,16 +1,20 @@
 import { writeLog } from "./writeLog.js";
 import { processFile } from "./processFile.js";
+import { ReconcileData } from "../types.js";
 
-export const publishToReconciliation = (filePath, id, language) => {
+/**
+  * @param {ReconcileData} reconcileData
+  */
+export const publishToReconciliation = (reconcileData) => {
   const log = {
-    id: id,
+    id: reconcileData.id,
     status: "processing",
     log: [],
   };
   writeLog(log);
   return new Promise((resolve, reject) => {
     try {
-      resolve(processFile(filePath, log, language));
+      resolve(processFile(reconcileData, log));
     } catch (error) {
       console.log("rejected");
       reject(error);

src/publishToReconciliation/processFile.js

@@ -1,6 +1,7 @@
 import { parseFile, sendData, deleteData } from "./handleData.js";
 import { writeLog } from "./writeLog.js";
 import { config } from "../config.js";
+import { ReconcileData } from "../types.js";
 
 const esIndex = config.es_index;
 
@@ -12,9 +13,13 @@ function HandleDataError(message, error, account, dataset) {
   this.dataset = dataset;
 }
 
-export const processFile = async (filePath, log, language) => {
+/**
+ * @param {ReconcileData} reconcileData
+ * @param {Object} log
+ */
+export const processFile = async (reconcileData, log) => {
   try {
-    const data = await parseFile(filePath, log);
+    const data = await parseFile(reconcileData, log);
     for await (const v of data) {
       // delete old data
       const responseDeleted = await deleteData(v.account, v.dataset);
@@ -46,7 +51,7 @@ export const processFile = async (filePath, log, language) => {
       // TODO improve this
       log.account = v.account;
       log.dataset = v.dataset;
-      log.language = language
+      log.language = reconcileData.language
       log.status = "success";
       log.reconcile_service_url = config.reconcile_service_url;
       writeLog(log);

src/types.js

@@ -0,0 +1,10 @@
+
+/**
+  * @typedef {Object} ReconcileData
+  * @property {string} filePath
+  * @property {string} fileUrl
+  * @property {string} account
+  * @property {string} id
+  * @property {string} language
+  */
+export const ReconcileData = {}