commit f92725ade7d9fcae1cb2b9fd3d93f31cbd02fbfc
parent 0c6595ee1305fa8e32ab3eb9e758ef8c489e46e3
Author: arjoonn <arjoonn@noreply.localhost>
Date: Sun, 15 Jan 2023 03:21:37 +0000
Plaintext human readable pipeline state (!21)
Branch auto created by JayporeCI
```jayporeci
╔ 🟢 : JayporeCI [344488fc95]
┏━ Docker
┃
┃ 🟢 : JciEnv [7ea888eb]
┃ 🟢 : Jci [60ab1e90]
┗━━━━━━━━━━━━━━━
┏━ Jobs
┃
┃ 🟢 : pytest [b94cd9bc]
┃ 🟢 : pylint [da675bc7]
┃ 🟢 : black [f1f170c2]
┗━━━━━━━━━━━━━━━
┏━ Publish
┃
┃ 🟢 : DockerHubJci [f41b85c9]
┃ 🟢 : DockerHubJcienv [bee57fc7]
┃ 🟢 : PublishDocs [8dcbfb07]
┃ 🟢 : PublishPypi [7b176648]
┗━━━━━━━━━━━━━━━
```
Co-authored-by: arjoonn sharma <arjoonn@midpathsoftware.com>
Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/21
Diffstat:
9 files changed, 232 insertions(+), 153 deletions(-)
diff --git a/cicd/cicd.py b/cicd/cicd.py
@@ -1,7 +1,7 @@
-from jaypore_ci import jci
+from jaypore_ci import jci, reporters
-with jci.Pipeline() as p:
+with jci.Pipeline(reporter=reporters.Text()) as p:
jcienv = f"jcienv:{p.remote.sha}"
with p.stage("Docker"):
p.job("JciEnv", f"docker build --target jcienv -t jcienv:{p.remote.sha} .")
diff --git a/jaypore_ci/interfaces.py b/jaypore_ci/interfaces.py
@@ -4,12 +4,23 @@ Defines interfaces for remotes and executors.
Currently only gitea and docker are supported as remote and executor
respectively.
"""
+from enum import Enum
class TriggerFailed(Exception):
"Failure to trigger a job"
+class Status(Enum):
+ "Each pipeline can be in any one of these statuses"
+ PENDING = 10
+ RUNNING = 30
+ FAILED = 40
+ PASSED = 50
+ TIMEOUT = 60
+ SKIPPED = 70
+
+
class Executor:
"""
It could be docker / podman / shell etc.
@@ -67,3 +78,15 @@ class Remote:
@classmethod
def from_env(cls):
raise NotImplementedError()
+
+
+class Reporter:
+ """
+ Something that allows us to report the status of a pipeline
+ """
+
+ def render(self, pipeline):
+ """
+ Render a report for the pipeline.
+ """
+ raise NotImplementedError()
diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py
@@ -3,56 +3,25 @@ The code submodule for Jaypore CI.
"""
import time
import os
-import re
-from enum import Enum
from itertools import product
-from collections import defaultdict, namedtuple
+from collections import defaultdict
from typing import List, Union, Callable
from contextlib import contextmanager
import structlog
import pendulum
-from jaypore_ci import remotes, executors
-from jaypore_ci.interfaces import Remote, Executor, TriggerFailed
-from jaypore_ci.logging import logger, jaypore_logs
+from jaypore_ci import remotes, executors, reporters
+from jaypore_ci.interfaces import Remote, Executor, Reporter, TriggerFailed, Status
+from jaypore_ci.logging import logger
TZ = "UTC"
__all__ = ["Pipeline", "Job"]
-class Status(Enum):
- "Each pipeline can be in any one of these statuses"
- PENDING = 10
- RUNNING = 30
- FAILED = 40
- PASSED = 50
- TIMEOUT = 60
- SKIPPED = 70
-
-
# All of these statuses are considered "finished" statuses
FIN_STATUSES = (Status.FAILED, Status.PASSED, Status.TIMEOUT, Status.SKIPPED)
-ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
-
-
-def __node_mod__(nodes):
- mod = 1
- if len(nodes) > 5:
- mod = 2
- if len(nodes) > 10:
- mod = 3
- return mod
-
-
-def __clean_logs__(logs):
- """
- Clean logs so that they don't have HTML/ANSI color codes in them.
- """
- for old, new in [("<", r"\<"), (">", r"\>"), ("`", '"'), ("\r", "\n")]:
- logs = logs.replace(old, new)
- return [line.strip() for line in ansi_escape.sub("", logs).split("\n")]
class Job: # pylint: disable=too-many-instance-attributes
@@ -122,7 +91,9 @@ class Job: # pylint: disable=too-many-instance-attributes
Status.TIMEOUT: "warning",
Status.SKIPPED: "warning",
}[self.pipeline.get_status()]
- self.pipeline.remote.publish(self.pipeline.render_report(), status)
+ self.pipeline.remote.publish(
+ self.pipeline.reporter.render(self.pipeline), status
+ )
def trigger(self):
"""
@@ -148,7 +119,7 @@ class Job: # pylint: disable=too-many-instance-attributes
job_name=self.name,
)
logs = job_run.stdout.decode()
- self.logs["stdout"] = __clean_logs__(logs)
+ self.logs["stdout"] = reporters.gitea.clean_logs(logs)
self.status = Status.FAILED
else:
self.logging().info("Trigger called but job already running")
@@ -170,7 +141,7 @@ class Job: # pylint: disable=too-many-instance-attributes
self.status = Status.RUNNING if not self.is_service else Status.PASSED
else:
self.status = Status.PASSED if exit_code == 0 else Status.FAILED
- self.logs["stdout"] = __clean_logs__(logs)
+ self.logs["stdout"] = reporters.gitea.clean_logs(logs)
if with_update_report:
self.update_report()
@@ -203,6 +174,7 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
self,
remote: Remote = None,
executor: Executor = None,
+ reporter: Reporter = None,
*,
graph_direction: str = "TB",
poll_interval: int = 1,
@@ -213,6 +185,7 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
self.should_pass_called = set()
self.remote = remote if remote is not None else remotes.gitea.Gitea.from_env()
self.executor = executor if executor is not None else executors.docker.Docker()
+ self.reporter = reporter if reporter is not None else reporters.gitea.Gitea()
self.graph_direction = graph_direction
self.poll_interval = poll_interval
self.executor.set_pipeline(self)
@@ -260,12 +233,13 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
for service in self.services:
service.check_job(with_update_report=False)
if service is not None:
- service.check_job()
- has_pending = True
+ service.check_job(with_update_report=False)
+ has_pending = False
for job in self.jobs.values():
job.check_job(with_update_report=False)
- if job.is_complete():
- has_pending = False
+ if not job.is_complete():
+ has_pending = True
+ else:
if job.status != Status.PASSED:
return Status.FAILED
return Status.PENDING if has_pending else Status.PASSED
@@ -282,113 +256,6 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
return "🔵"
return "🟡"
- def render_report(self):
- """
- Returns a markdown report for a given pipeline.
-
- It will include a mermaid graph and a collapsible list of logs for each
- job.
- """
- return f"""
-<details>
- <summary>JayporeCi: {self.get_status_dot()} {self.remote.sha[:10]}</summary>
-
-{self.__render_graph__()}
-{self.__render_logs__()}
-
-</details>"""
-
- def __render_graph__(self) -> str: # pylint: disable=too-many-locals
- """
- Render a mermaid graph given the jobs in the pipeline.
- """
- st_map = {
- Status.PENDING: "pending",
- Status.RUNNING: "running",
- Status.FAILED: "failed",
- Status.PASSED: "passed",
- Status.TIMEOUT: "timeout",
- Status.SKIPPED: "skipped",
- }
- mermaid = f"""
-```mermaid
-flowchart {self.graph_direction}
-"""
- for stage in self.stages:
- nodes, edges = set(), set()
- for job in self.jobs.values():
- if job.stage != stage:
- continue
- nodes.add(job.name)
- edges |= {(p, job.name) for p in job.parents}
- mermaid += f"""
- subgraph {stage}
- direction {self.graph_direction}
- """
- ref = {n: f"{stage}_{i}" for i, n in enumerate(nodes)}
- # If there are too many nodes, scatter them with different length arrows
- mod = __node_mod__([n for n in nodes if not self.jobs[n].parents])
- for i, n in enumerate(nodes):
- n = self.jobs[n]
- if n.parents:
- continue
- arrow = "." * ((i % mod) + 1)
- arrow = f"-{arrow}->"
- mermaid += f"""
- s_{stage}(( )) {arrow} {ref[n.name]}({n.name}):::{st_map[n.status]}"""
- mod = __node_mod__([n for n in nodes if self.jobs[n].parents])
- for i, (a, b) in enumerate(edges):
- a, b = self.jobs[a], self.jobs[b]
- arrow = "." * ((i % mod) + 1)
- arrow = f"-{arrow}->"
- mermaid += f"""
- {ref[a.name]}({a.name}):::{st_map[a.status]} {arrow} {ref[b.name]}({b.name}):::{st_map[b.status]}"""
- mermaid += """
- end
- """
- for s1, s2 in zip(self.stages, self.stages[1:]):
- mermaid += f"""
- {s1} ---> {s2}
- """
- mermaid += """
-
- classDef pending fill:#aaa, color:black, stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
- classDef skipped fill:#aaa, color:black, stroke:black,stroke-width:2px;
- classDef assigned fill:#ddd, color:black, stroke:black,stroke-width:2px;
- classDef running fill:#bae1ff,color:black,stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
- classDef passed fill:#88d8b0, color:black, stroke:black;
- classDef failed fill:#ff6f69, color:black, stroke:black;
- classDef timeout fill:#ffda9e, color:black, stroke:black;
-``` """
- return mermaid
-
- def __render_logs__(self):
- """
- Collect all pipeline logs and render into a single collapsible text.
- """
- all_logs = []
- fake_job = namedtuple("fake_job", "name logs")(
- "JayporeCi",
- {"stdout": __clean_logs__("\n".join(jaypore_logs))},
- )
- for job in [fake_job] + list(self.jobs.values()):
- job_log = []
- for logname, stream in job.logs.items():
- job_log += [f"============== {logname} ============="]
- job_log += [line.strip() for line in stream]
- if job_log:
- all_logs += [
- "- <details>",
- f" <summary>Logs: {job.name}</summary>",
- "",
- " ```",
- *[f" {line}" for line in job_log],
- " ```",
- "",
- " </details>",
- ]
- return "\n".join(all_logs)
-
def job(
self,
name: str,
diff --git a/jaypore_ci/remotes/gitea.py b/jaypore_ci/remotes/gitea.py
@@ -139,7 +139,7 @@ class Gitea(Remote): # pylint: disable=too-many-instance-attributes
body = (line for line in body.split("\n"))
prefix = []
for line in body:
- if "<summary>JayporeCi" in line:
+ if "```jayporeci" in line:
prefix = prefix[:-1]
break
prefix.append(line)
diff --git a/jaypore_ci/reporters/__init__.py b/jaypore_ci/reporters/__init__.py
@@ -0,0 +1,3 @@
+from .gitea import Gitea
+from .mock import Mock
+from .text import Text
diff --git a/jaypore_ci/reporters/gitea.py b/jaypore_ci/reporters/gitea.py
@@ -0,0 +1,134 @@
+import re
+from collections import namedtuple
+from jaypore_ci.interfaces import Reporter, Status
+from jaypore_ci.logging import jaypore_logs
+
+
+def __node_mod__(nodes):
+ mod = 1
+ if len(nodes) > 5:
+ mod = 2
+ if len(nodes) > 10:
+ mod = 3
+ return mod
+
+
+ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+
+
+def clean_logs(logs):
+ """
+ Clean logs so that they don't have HTML/ANSI color codes in them.
+ """
+ for old, new in [("<", r"\<"), (">", r"\>"), ("`", '"'), ("\r", "\n")]:
+ logs = logs.replace(old, new)
+ return [line.strip() for line in ansi_escape.sub("", logs).split("\n")]
+
+
+class Gitea(Reporter):
+ def render(self, pipeline):
+ """
+ Returns a markdown report for a given pipeline.
+
+ It will include a mermaid graph and a collapsible list of logs for each
+ job.
+ """
+ return f"""
+<details>
+ <summary>JayporeCi: {pipeline.get_status_dot()} {pipeline.remote.sha[:10]}</summary>
+
+{self.__render_graph__(pipeline)}
+{self.__render_logs__(pipeline)}
+
+</details>"""
+
+ def __render_graph__(self, pipeline) -> str: # pylint: disable=too-many-locals
+ """
+ Render a mermaid graph given the jobs in the pipeline.
+ """
+ st_map = {
+ Status.PENDING: "pending",
+ Status.RUNNING: "running",
+ Status.FAILED: "failed",
+ Status.PASSED: "passed",
+ Status.TIMEOUT: "timeout",
+ Status.SKIPPED: "skipped",
+ }
+ mermaid = f"""
+```mermaid
+flowchart {pipeline.graph_direction}
+"""
+ for stage in pipeline.stages:
+ nodes, edges = set(), set()
+ for job in pipeline.jobs.values():
+ if job.stage != stage:
+ continue
+ nodes.add(job.name)
+ edges |= {(p, job.name) for p in job.parents}
+ mermaid += f"""
+ subgraph {stage}
+ direction {pipeline.graph_direction}
+ """
+ ref = {n: f"{stage}_{i}" for i, n in enumerate(nodes)}
+ # If there are too many nodes, scatter them with different length arrows
+ mod = __node_mod__([n for n in nodes if not pipeline.jobs[n].parents])
+ for i, n in enumerate(nodes):
+ n = pipeline.jobs[n]
+ if n.parents:
+ continue
+ arrow = "." * ((i % mod) + 1)
+ arrow = f"-{arrow}->"
+ mermaid += f"""
+ s_{stage}(( )) {arrow} {ref[n.name]}({n.name}):::{st_map[n.status]}"""
+ mod = __node_mod__([n for n in nodes if pipeline.jobs[n].parents])
+ for i, (a, b) in enumerate(edges):
+ a, b = pipeline.jobs[a], pipeline.jobs[b]
+ arrow = "." * ((i % mod) + 1)
+ arrow = f"-{arrow}->"
+ mermaid += f"""
+ {ref[a.name]}({a.name}):::{st_map[a.status]} {arrow} {ref[b.name]}({b.name}):::{st_map[b.status]}"""
+ mermaid += """
+ end
+ """
+ for s1, s2 in zip(pipeline.stages, pipeline.stages[1:]):
+ mermaid += f"""
+ {s1} ---> {s2}
+ """
+ mermaid += """
+
+ classDef pending fill:#aaa, color:black, stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
+ classDef skipped fill:#aaa, color:black, stroke:black,stroke-width:2px;
+ classDef assigned fill:#ddd, color:black, stroke:black,stroke-width:2px;
+ classDef running fill:#bae1ff,color:black,stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
+ classDef passed fill:#88d8b0, color:black, stroke:black;
+ classDef failed fill:#ff6f69, color:black, stroke:black;
+ classDef timeout fill:#ffda9e, color:black, stroke:black;
+``` """
+ return mermaid
+
+ def __render_logs__(self, pipeline):
+ """
+ Collect all pipeline logs and render into a single collapsible text.
+ """
+ all_logs = []
+ fake_job = namedtuple("fake_job", "name logs")(
+ "JayporeCi",
+ {"stdout": clean_logs("\n".join(jaypore_logs))},
+ )
+ for job in [fake_job] + list(pipeline.jobs.values()):
+ job_log = []
+ for logname, stream in job.logs.items():
+ job_log += [f"============== {logname} ============="]
+ job_log += [line.strip() for line in stream]
+ if job_log:
+ all_logs += [
+ "- <details>",
+ f" <summary>Logs: {job.name}</summary>",
+ "",
+ " ```",
+ *[f" {line}" for line in job_log],
+ " ```",
+ "",
+ " </details>",
+ ]
+ return "\n".join(all_logs)
diff --git a/jaypore_ci/reporters/mock.py b/jaypore_ci/reporters/mock.py
@@ -0,0 +1,6 @@
+from jaypore_ci.interfaces import Reporter
+
+
+class Mock(Reporter):
+ def render(self, pipeline):
+ return f"{pipeline}"
diff --git a/jaypore_ci/reporters/text.py b/jaypore_ci/reporters/text.py
@@ -0,0 +1,46 @@
+from jaypore_ci.interfaces import Reporter, Status
+
+
+class Text(Reporter):
+ def render(self, pipeline):
+ """
+ Returns a human readable report for a given pipeline.
+ """
+ st_map = {
+ Status.RUNNING: "🔵",
+ Status.FAILED: "🔴",
+ Status.PASSED: "🟢",
+ }
+ max_name = max(len(job.name) for job in pipeline.jobs.values())
+ max_name = max(max_name, len("jayporeci"))
+ name = ("JayporeCI" + " " * max_name)[:max_name]
+ graph = [
+ "",
+ "```jayporeci",
+ f"╔ {pipeline.get_status_dot()} : {name} [{pipeline.remote.sha[:10]}]",
+ ]
+ for stage in pipeline.stages:
+ nodes, edges = set(), set()
+ for job in pipeline.jobs.values():
+ if job.stage != stage:
+ continue
+ nodes.add(job.name)
+ edges |= {(p, job.name) for p in job.parents}
+ if not nodes:
+ continue
+ graph += [f"┏━ {stage}", "┃"]
+ for n in sorted(
+ nodes, key=lambda x: len(pipeline.jobs[x].parents)
+ ): # Fewer parents first
+ n = pipeline.jobs[n]
+ name = (n.name + " " * max_name)[:max_name]
+ status = st_map.get(n.status, "🟡")
+ run_id = f"{n.run_id}"[:8] if n.run_id is not None else ""
+ if n.parents:
+ graph += [f"┃ {status} : {name} [{run_id:<8}] ← {n.parents}"]
+ else:
+ graph += [f"┃ {status} : {name} [{run_id:<8}]"]
+ graph += ["┗━━━━━━━━━━━━━━━"]
+ graph += ["```"]
+ graph = "\n".join(graph)
+ return f"\n{graph}"
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "jaypore_ci"
-version = "0.1.3"
+version = "0.1.5"
description = ""
authors = ["arjoonn sharma <arjoonn.94@gmail.com>"]
