CORD-19_dataset_author_counts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import re\n",
    "from bs4 import BeautifulSoup"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "url = 'https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/historical_releases.html'"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text)\n",
    "latest_date = soup.td.get_text()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "file_list = []\n",
    "for tag in soup.find_all(href=re.compile(\"tar\")):\n",
    "    file_name = tag.get_text().replace('.tar.gz', '').split('_')[1]\n",
    "    file_list.append(file_name)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "dataset_statistics = {}\n",
    "for i in file_list:\n",
    "    df = pd.read_csv('https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/' + i + '/metadata.csv', parse_dates=['publish_time'])\n",
    "    df['author_list'] = df['authors'].astype(str).apply(lambda x: x.split(';'))\n",
    "    df['author_count'] = df['author_list'].str.len()\n",
    "    dataset_statistics[i] = df['author_count'].describe().tolist()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "describe = ['Article Count', 'Mean Author Count', 'Std Author Count', 'Min Author Count', '25% Percentile', '50% Percentile', '75% Percentile', 'Max Author Count']\n",
    "data = pd.DataFrame.from_dict(dataset_statistics, orient='index')\n",
    "data.columns = describe"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "data.to_csv('~/github-actions-test/data/CORD-19-historical-releases.csv')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.10",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.10 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "9a4465037f6805cb85f76bcb677ef345a46c4f4c12d1783643b2731ec4d2c4a1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}