Add Documentation (#2)

* WIP: added docs until encoders * Further create docs * Fixed upgrade to series and column name for concat * Adapt NotImplementedError * Fix failing tests * Add meta to qgramsblocker * Finalized docs,linting,type problems * Add architecture logo cleanup som deps * Add missing picture * Free ubuntu disk space * Remove failing flag and missing import
dobraczka · Oct 10, 2023 · b2d3e50 · b2d3e50
1 parent c5dd450
commit b2d3e50
Show file tree

Hide file tree

Showing 47 changed files with 2,322 additions and 1,210 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,15 +14,17 @@ on:
 jobs:
   build:
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.8, "3.10"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
+      fail-fast: True
     steps:
     - uses: actions/checkout@v3
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
     - name: Install poetry
       run: pipx install poetry
     - name: Set up Python ${{ matrix.python-version }}

diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml
@@ -12,6 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
     - name: Install poetry
       run: pipx install poetry
     - name: Setup python
@@ -33,6 +39,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
     - name: Install poetry
       run: pipx install poetry
     - name: Setup python

diff --git a/README.md b/README.md
@@ -3,10 +3,15 @@
 </p>
 <h2 align="center"> klinker</h2>
 
-<!-- <p align="center"> -->
-<!-- <a href="https://github.com/dobraczka/klinker/actions/workflows/main.yml"><img alt="Actions Status" src="https://github.com/dobraczka/klinker/actions/workflows/main.yml/badge.svg?branch=main"></a> -->
-<!-- <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a> -->
-<!-- </p> -->
+<p align="center">
+<a href="https://github.com/dobraczka/klinker/actions/workflows/main.yml"><img alt="Actions Status" src="https://github.com/dobraczka/klinker/actions/workflows/main.yml/badge.svg?branch=main"></a>
+<a href='https://klinker.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/klinker/badge/?version=latest' alt='Documentation Status' /></a>
+<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
+</p>
+
+<p align="center">
+<img src="https://github.com/dobraczka/klinker/raw/main/docs/assets/KlinkerArchitectureNoLogo.png" alt="klinker overview", width=200/>
+</p>
 
 Installation
 ============

diff --git a/docs/assets/KlinkerArchitectureNoLogo.png b/docs/assets/KlinkerArchitectureNoLogo.png
diff --git a/docs/index.md b/docs/index.md
@@ -1,12 +1,90 @@
 <p align="center">
 <img src="https://github.com/dobraczka/klinker/raw/main/docs/assets/logo.png" alt="klinker logo", width=200/>
-<h2 align="center"> strawman</h2>
 </p>
+<h2 align="center"> klinker</h2>
 
 <p align="center">
 <a href="https://github.com/dobraczka/klinker/actions/workflows/main.yml"><img alt="Actions Status" src="https://github.com/dobraczka/klinker/actions/workflows/main.yml/badge.svg?branch=main"></a>
-<a href='https://strawman.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/klinker/badge/?version=latest' alt='Documentation Status' /></a>
-<a href="https://codecov.io/gh/dobraczka/klinker"><img src="https://codecov.io/gh/dobraczka/klinker/branch/main/graph/badge.svg"/></a>
-<a href="https://pypi.org/project/klinker"/><img alt="Stable python versions" src="https://img.shields.io/pypi/pyversions/klinker"></a>
+<a href='https://klinker.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/klinker/badge/?version=latest' alt='Documentation Status' /></a>
 <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
 </p>
+
+<p align="center">
+<img src="https://github.com/dobraczka/klinker/raw/main/docs/assets/KlinkerArchitectureNoLogo.png" alt="klinker overview", width=200/>
+</p>
+
+Installation
+============
+Clone the repo and change into the directory:
+
+```bash
+git clone https://github.com/dobraczka/klinker.git
+cd klinker
+```
+
+For usage with GPU create a [micromamba](https://mamba.readthedocs.io/en/latest/micromamba-installation.html) environment:
+
+```bash
+micromamba env create -n klinker-conda --file=klinker-conda.yaml
+```
+
+Activate it and install the remaining dependencies:
+```
+mamba activate klinker-conda
+pip install -e .
+```
+
+Alternatively if you don't intend to utilize a GPU you can install it in a virtual environment:
+```
+python -m venv klinker-env
+source klinker-env/bin/activate
+pip install -e .[all]
+```
+
+or via [poetry](https://python-poetry.org/docs/):
+```
+poetry install
+```
+
+Usage
+=====
+Load a dataset:
+```python
+from sylloge import MovieGraphBenchmark
+from klinker.data import KlinkerDataset
+
+ds = KlinkerDataset.from_sylloge(MovieGraphBenchmark(graph_pair="tmdb-tvdb"))
+```
+
+Create blocks and write to parquet:
+
+```python
+from klinker.blockers import SimpleRelationalTokenBlocker
+
+blocker = SimpleRelationalTokenBlocker()
+blocks = blocker.assign(left=ds.left, right=ds.right, left_rel=ds.left_rel, right_rel=ds.right_rel)
+blocks.to_parquet("tmdb-tvdb-tokenblocked")
+```
+
+Read blocks from parquet and evaluate:
+```python
+from klinker import KlinkerBlockManager
+from klinker.eval_metrics import Evaluation
+
+kbm = KlinkerBlockManager.read_parqet("tmdb-tvdb-tokenblocked")
+ev = Evaluation.from_dataset(blocks=kbm, dataset=ds)
+```
+
+Reproduce Experiments
+=====================
+
+The `experiment.py` has commands for datasets and blockers. You can use `python experiment.py --help` to show the available commands. Subcommands can also offer help e.g. `python experiment.py gcn-blocker --help`.
+
+You have to use a dataset command before a blocker command.
+
+For example if you used micromamba for installation:
+```bash
+micromamba run -n klinker-conda python experiment.py movie-graph-benchmark-dataset --graph-pair "tmdb-tvdb" relational-token-blocker
+```
+
+This would be similar to the steps described in the above usage section.
diff --git a/encoder_experiment.py b/encoder_experiment.py
@@ -17,7 +17,7 @@
     frame_encoder_resolver,
 )
 from klinker.encoders.deepblocker import DeepBlockerFrameEncoder
-from klinker.eval_metrics import Evaluation
+from klinker.eval import Evaluation
 
 
 def _load_or_create_path(

diff --git a/experiment.py b/experiment.py
@@ -50,7 +50,7 @@
     TokenizedFrameEncoder,
     tokenized_frame_encoder_resolver,
 )
-from klinker.eval_metrics import Evaluation
+from klinker.eval import Evaluation
 from klinker.trackers import ConsoleResultTracker, ResultTracker, WANDBResultTracker
 
 logger = logging.getLogger("KlinkerExperiment")

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -43,8 +43,9 @@ plugins:
   - mkdocstrings:
       handlers:
         python:
-          selection:
-            docstring_style: google
+          docstring_style: google
+          docstring_options:
+            trim_doctest_flags: true
 
 extra:
   social:

diff --git a/noxfile.py b/noxfile.py
@@ -5,6 +5,12 @@
 def tests(session: Session) -> None:
     args = session.posargs or ["--cov", "--cov-report=xml"]
     session.install(".[all]")
+    session.install(
+        "torch-scatter", "-f", "https://data.pyg.org/whl/torch-2.0.0+cpu.html"
+    )
+    session.install(
+        "torch-sparse", "-f", "https://data.pyg.org/whl/torch-2.0.0+cpu.html"
+    )
     session.install("strawman")
     session.install("pytest")
     session.install("pytest-cov")
@@ -35,10 +41,9 @@ def style_checking(session: Session) -> None:
         "flake8-print",
         "flake8-black",
         "flake8-bugbear",
-        "darglint",
         "pydocstyle",
     )
-    session.run("pflake8", "--docstring-style", "sphinx", *args)
+    session.run("pflake8", *args)
 
 
 @session()