Jaypore CI

commit ee33122007a4457fc4a46ada5c2df430a5cf5c9b
parent c6a97f2ce8e60bc326623849b68f2356d4ea2710
Author: arjoonn <arjoonn@noreply.localhost>
Date:   Thu, 30 Mar 2023 08:54:25 +0000

Prune networks during old container prune (!78)

Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/78
╔ 🔴 : JayporeCI       [sha 15fd3e2d5b]
┏━ build-and-test
┃
┃ 🟢 : JciEnv          [7589e605]   0:12
┃ 🟢 : Jci             [742ecb75]   0:18            ❮-- ['JciEnv']
┃ 🟢 : black           [68992f99]   0: 0            ❮-- ['JciEnv']
┃ 🟢 : install-test    [717173b8]   0: 0            ❮-- ['JciEnv']
┃ 🟢 : pylint          [642eb24b]   0:10            ❮-- ['JciEnv']
┃ 🟢 : pytest          [6a8e4f34]   0:28 Cov: 89%   ❮-- ['JciEnv']
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Publish
┃
┃ 🟢 : DockerHubJci    [eaae4d95]   0:52
┃ 🟢 : DockerHubJcienv [faa653f9]   0:52
┃ 🟢 : PublishDocs     [0fbaeb6e]   0:39
┃ 🔴 : PublishPypi     [0a9dae01]   0: 4 v0.2.30
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛

Diffstat:
M
cicd/build_and_publish_docs.sh
 | 
5
+++++
M
docs/source/conf.py
 | 
4
++++
M
docs/source/index.rst
 | 
3
+--
M
jaypore_ci/changelog.py
 | 
9
+++++++++
M
jaypore_ci/executors/docker.py
 | 
17
+++++++++++++++--
M
jaypore_ci/jci.py
 | 
2
+-

6 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/cicd/build_and_publish_docs.sh b/cicd/build_and_publish_docs.sh
@@ -5,10 +5,15 @@ set -o nounset
 set -o pipefail
 
 build() {
+    echo "Cleaning docs build"
+    touch docs/build
+    rm -rf docs/build && mkdir -p docs/build
+    ls -al docs
     echo "Building docs"
     sphinx-apidoc -o docs/source/reference ./jaypore_ci
     (python3 cicd/render_changelog.py >> docs/source/index.rst)
     sphinx-build docs/source/ docs/build
+    sphinx-build docs/source/ docs/build -b coverage
 
     # Create pre-push for repo
     PREPUSH=docs/build/pre-push.sh
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -18,6 +18,9 @@ extensions = [
     "sphinx.ext.autosummary",
     "sphinx_rtd_theme",
     "sphinxcontrib.mermaid",
+    "sphinx.ext.coverage",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.todo",
 ]
 
 templates_path = ["_templates"]
@@ -51,3 +54,4 @@ html_theme_options = {
     # "titles_only": False,
 }
 master_doc = "contents"
+todo_include_todos = True
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -378,7 +378,7 @@ that will be used to run your job, like when you want to pass `extra_hosts` or
 `device_requests` to the container.
 
 To do such things you can use the `executor_kwargs` argument while defining the
-job using :method:`~jaypore_ci.jci.Pipeline.job`. Anything that you pass to
+job using :meth:`~jaypore_ci.jci.Pipeline.job`. Anything that you pass to
 this dictionary will be handed off to `Docker-py
 <https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run>`_
 and so you can use anything that is mentioned in that documentation.
@@ -421,4 +421,3 @@ Reference
 Changelog
 =========
 
-
diff --git a/jaypore_ci/changelog.py b/jaypore_ci/changelog.py
@@ -6,6 +6,15 @@ CHANGE = "⚙️"
 BUGFIX = "🐞"
 
 version_map = {
+    V("0.2.31"): {
+        "changes": [
+            (
+                f"{NEW}: Old networks will also be removed automatically for "
+                "jobs that are older than a week."
+            ),
+        ],
+        "instructions": [],
+    },
     V("0.2.30"): {
         "changes": [
             (
diff --git a/jaypore_ci/executors/docker.py b/jaypore_ci/executors/docker.py
@@ -61,21 +61,34 @@ class Docker(Executor):
 
     def delete_old_containers(self):
         a_week_back = pendulum.now().subtract(days=7)
+        pipe_ids_removed = set()
         for container in tqdm(
             self.docker.containers.list(filters={"status": "exited"}),
             desc="Removing jobs older than a week",
         ):
             if "jayporeci_" not in container.name:
                 continue
+            if "__job__" in container.name:
+                pipe_ids_removed.add(
+                    container.name.split("__job__")[1].split("__", 1)[0]
+                )
             finished_at = pendulum.parse(container.attrs["State"]["FinishedAt"])
             if finished_at <= a_week_back:
                 container.remove(v=True)
+        for network in tqdm(
+            self.docker.networks.list(
+                names=[self.get_net(pipe_id=pipe_id) for pipe_id in pipe_ids_removed]
+            ),
+            desc="Removing related networks",
+        ):
+            network.remove()
 
-    def get_net(self):
+    def get_net(self, *, pipe_id=None):
         """
         Return a network name based on what the curent pipeline is.
         """
-        return f"jayporeci__net__{self.pipe_id}" if self.pipe_id is not None else None
+        pipe_id = pipe_id if pipe_id is not None else self.pipe_id
+        return f"jayporeci__net__{pipe_id}" if pipe_id is not None else None
 
     def create_network(self):
         """
diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py
@@ -35,7 +35,7 @@ FIN_STATUSES = (Status.FAILED, Status.PASSED, Status.TIMEOUT, Status.SKIPPED)
 PREFIX = "JAYPORE_"
 
 # Check if we need to upgrade Jaypore CI
-def ensure_version_is_correct():
+def ensure_version_is_correct() -> None:
     """
     Ensure that the version of Jaypore CI that is running, the code inside
     cicd.py, and pre-push.sh are at compatible versions.