Merge pull request #31 from nbonfils/fix-inconsistent-fetch

Fix inconsistent fetch of works

Author: nbonfils

src/index.html

@@ -75,14 +75,8 @@ <h1 class="center" >Bibliograph 2</h1>
 	  <template x-if="tab === 'url'">
 	    <form @submit.prevent="
 				   loading('corpus', 'Fetching corpus from OpenAlex API');
-				   works = await fetchWorksFromUrl(apiUrl, maxWorks);
-				   done();
-				   loading('filters', 'Processing corpus data');
-				   setTimeout(() => {
-				     data = processWorks(works);
-				     filters = getFilters(data);
-				     done();
-				   }, 10);" >
+				   ({count, works} = await fetchWorksFromUrl(apiUrl, maxWorks));
+				   done();" >
 	      <textarea id="apiurl"
 			class="card w-100"
 			placeholder="Paste the OpenAlex API URL here..."

src/lib/fetch.js

@@ -19,6 +19,28 @@ const toQueryParams = (params) => {
   return queryParams;
 };
 
+const fetchWorksCount = async (filter) => {
+  let count = 0;
+  try {
+    const response = await fetch(
+      "https://api.openalex.org/works?" + new URLSearchParams({
+	filter,
+	select: "id",
+        mailto: `****@****.com`,
+        "per-page": 1,
+	page: 1,
+      }).toString());
+    if (!response.ok) {
+      throw new Error("Network response was not 200 OK");
+    }
+    const data = await response.json();
+    count = data.meta.count;
+  } catch (e) {
+    console.error(`Error while fetching works count:\n\t${e}`);
+  }
+  return count;
+};
+
 export const fetchWorks = async (params, maxWorks) => {
   const qp = toQueryParams(params);
   let works = [];
@@ -27,7 +49,6 @@ export const fetchWorks = async (params, maxWorks) => {
   if (qp.title || qp.titleabs ||
       qp.titleabsfull || qp.concepts) {
     const filters = [];
-    const numReq = Math.ceil(maxWorks / perPage);
 
     if (qp.fromYear) {
       filters.push(`from_publication_date:${qp.fromYear}-01-01`);
@@ -50,6 +71,9 @@ export const fetchWorks = async (params, maxWorks) => {
       filters.push(`concepts.id:${qp.concepts}`);
     }
 
+    count = await fetchWorksCount(filters.join(","));
+    const numReq = Math.ceil(Math.min(count, maxWorks) / perPage);
+
     works = await Promise.all([...Array(numReq).keys()].map(async (i) => {
       let data = {};
       try {
@@ -63,10 +87,9 @@ export const fetchWorks = async (params, maxWorks) => {
 	    page: i+1,
 	  }).toString());
 	if (!response.ok) {
-	  throw new Error("Network response was not OK");
+	  throw new Error("Network response was not 200 OK");
 	}
 	data = await response.json();
-	count = data.meta.count;
       } catch (e) {
 	console.error(`Error while fetching works:\n\t${e}`);
       }
@@ -85,26 +108,27 @@ export const checkApiUrl = (url) => {
 
 export const fetchWorksFromUrl = async (url, maxWorks) => {
   let works = [];
-  const numReq = Math.ceil(maxWorks / perPage);
 
   const urlObj = new URL(url);
   const filter = urlObj.searchParams.get("filter");
-  const sort = urlObj.searchParams.get("sort");
+
+  const count = await fetchWorksCount(filter);
+  const numReq = Math.ceil(Math.min(count, maxWorks) / perPage);
 
   works = await Promise.all([...Array(numReq).keys()].map(async (i) => {
     let data = {};
     try {
       const response = await fetch(
         "https://api.openalex.org/works?" + new URLSearchParams({
 	  filter,
-	  sort,
+	  sort: 'cited_by_count:desc',
 	  select: "id,title,publication_year,primary_location,authorships,concepts,locations,grants,referenced_works,cited_by_count",
           mailto: `****@****.com`,
           "per-page": perPage,
 	  page: i+1,
 	}).toString());
       if (!response.ok) {
-	throw new Error("Network response was not OK");
+	throw new Error("Network response was not 200 OK");
       }
       data = await response.json();
     } catch (e) {
@@ -114,7 +138,7 @@ export const fetchWorksFromUrl = async (url, maxWorks) => {
   }));
 
   works = works.flat().slice(0, maxWorks).filter(work => work);
-  return works;
+  return {count, works};
 };
 
 export const fetchRefsLabels = async (openalexIds, maxRefs = 500) => {