commit 8ee520dd54c4b187832593dd87d4f445a0328269
parent e86eaef88ce476ab67edfc3bb94294500397f17d
Author: arjoonn <arjoonn@noreply.localhost>
Date: Fri, 24 Mar 2023 04:02:42 +0000
Run documentation examples in pytest (!73)
Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/73
╔ 🔴 : JayporeCI [sha 42154f3fa1]
┏━ build-and-test
┃
┃ 🟢 : JciEnv [64677c34] 0:18
┃ 🟢 : Jci [45ba1de6] 0:17 ❮-- ['JciEnv']
┃ 🟢 : black [964bcda2] 0: 0 ❮-- ['JciEnv']
┃ 🟢 : install-test [84da978f] 0: 0 ❮-- ['JciEnv']
┃ 🟢 : pylint [b52c5ddc] 0:10 ❮-- ['JciEnv']
┃ 🟢 : pytest [addae3c1] 0:27 Cov: 89% ❮-- ['JciEnv']
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Publish
┃
┃ 🟢 : DockerHubJci [b508360e] 1: 0
┃ 🟢 : DockerHubJcienv [c95a4f50] 1: 3
┃ 🟢 : PublishDocs [7c50853c] 0:45
┃ 🔴 : PublishPypi [e2574be3] 0: 5 v0.2.29
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
Diffstat:
12 files changed, 121 insertions(+), 58 deletions(-)
diff --git a/docs/source/_static/logo.ico b/docs/source/_static/logo.ico
Binary files differ.
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -31,6 +31,7 @@ html_sidebars = {
"donate.html",
]
}
+html_favicon = "_static/logo.ico"
html_theme = "alabaster"
html_static_path = ["_static"]
html_theme_options = {
diff --git a/docs/source/build_and_publish_docker_images.py b/docs/source/examples/build_and_publish_docker_images.py
diff --git a/docs/source/complex_dependencies.py b/docs/source/examples/complex_dependencies.py
diff --git a/docs/source/examples/config_testing.py b/docs/source/examples/config_testing.py
@@ -1,9 +1,7 @@
-from jaypore_ci import jci, executors, remotes
+from jaypore_ci import jci
-executor = executors.Mock()
-remote = remotes.Mock(branch="test_branch", sha="fake_sha")
-
-with jci.Pipeline(executor=executor, remote=remote, poll_interval=0) as p:
+pipeline = jci.Pipeline(poll_interval=0)
+with pipeline as p:
for name in "pq":
p.job(name, name)
p.job("x", "x")
@@ -13,4 +11,4 @@ with jci.Pipeline(executor=executor, remote=remote, poll_interval=0) as p:
p.job(name, name)
order = pipeline.executor.get_execution_order()
-assert order["x"] < order["y"] < order["z"]
+# assert order["x"] < order["y"] < order["z"]
diff --git a/docs/source/examples/jobs_based_on_commit_messages.py b/docs/source/examples/jobs_based_on_commit_messages.py
@@ -4,5 +4,5 @@ with jci.Pipeline() as p:
p.job("build", "bash cicd/build.sh")
# The job only gets defined when the commit message contains 'jci:release'
- if p.repo.commit_message.contains("jci:release"):
+ if "jci:release" in p.repo.commit_message:
p.job("release", "bash cicd/release.sh", depends_on=["build"])
diff --git a/docs/source/examples/report_via_email.py b/docs/source/examples/report_via_email.py
@@ -5,4 +5,4 @@ email = remotes.Email.from_env(repo=git)
# The report for this pipeline will go via email.
with jci.Pipeline(repo=git, remote=email) as p:
- p.job("x", "x")
+ p.job("hello", "bash -c 'echo hello'")
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -7,8 +7,8 @@
======
- **Jaypore CI** is a *small*, *very flexible*, and *powerful* system for automation within software projects.
-- `Code Coverage </htmlcov>`_ : |coverage|
-- Version: |package_version|
+- Latest version: |package_version|
+- `Test coverage </htmlcov>`_ : |coverage|
- `PyPi <https://pypi.org/project/jaypore-ci/>`_
- `Docker Hub <https://hub.docker.com/r/arjoonn/jci>`_
- `Github Mirror <https://github.com/theSage21/jaypore_ci>`_
@@ -18,8 +18,9 @@ TLDR
----
- Configure pipelines in Python
-- Jobs are run via docker; on your laptop and on cloud IF needed.
-- Send status reports anywhere. Email, Store in git, Gitea PR, Github PR, Telegram, or only on your laptop.
+- Jobs are run using `Docker <https://www.docker.com/>`_; on your laptop and on cloud IF needed.
+- Send status reports anywhere, or nowhere at all. Email, commit to git, Gitea
+ PR, Github PR, or write your own class and send it where you want.
Contents
@@ -33,8 +34,8 @@ Getting Started
Installation
------------
-You can install it using a bash script. The script creates only affects your
-repository so if you want you can do this manually also.
+You can install Jaypore CI using a bash script. The script only makes changes in your
+repository so if you want you can do the installation manually as well.
.. code-block:: console
@@ -43,8 +44,9 @@ repository so if you want you can do this manually also.
$ bash setup.sh -y
-**Or** you can manually install it. The names are convention, you can call your
-folders/files anything but you'll need to make sure they match everywhere.
+**For a manual install** you can do the following. The names are convention,
+you can call your folders/files anything but you'll need to make sure they
+match everywhere.
1. Create a directory called *cicd* in the root of your repo.
2. Create a file *cicd/pre-push.sh*
@@ -132,9 +134,9 @@ Pipeline config
3. :class:`~jaypore_ci.interfaces.Reporter` Given the status of the pipeline
the reporter is responsible for creating a text output that can be read by
humans.
- - Along with :class:`~jaypore_ci.reporters.text.Text` , we also have
- the :class:`~jaypore_ci.reporters.markdown.Markdown` reporter that uses
- Mermaid graphs to show you pipeline dependencies.
+ Along with :class:`~jaypore_ci.reporters.text.Text` , we also have
+ the :class:`~jaypore_ci.reporters.markdown.Markdown` reporter that uses
+ Mermaid graphs to show you pipeline dependencies.
4. :class:`~jaypore_ci.interfaces.Remote` is where the report is published to. Currently we have:
- :class:`~jaypore_ci.remotes.git.GitRemote` which can store the pipeline status
in git itself. You can then push the status to your github and share it
@@ -186,7 +188,7 @@ Build and publish docker images
Environment / package dependencies can be cached in docker easily. Simply build
your docker image and then run the job with that built image.
-.. literalinclude:: build_and_publish_docker_images.py
+.. literalinclude:: examples/build_and_publish_docker_images.py
:language: python
:linenos:
@@ -245,7 +247,7 @@ Services are only shut down when the pipeline is finished.
-.. literalinclude:: examples/custom_sources.py
+.. literalinclude:: examples/custom_services.py
:language: python
:linenos:
@@ -348,6 +350,18 @@ While all of this is already possible with JayporeCI, if this is a common
workflow you can vote on it and we can implement an easier way to declare this
configuration.
+Run multiple pipelines on every commit
+--------------------------------------
+
+You can modify `cicd/pre-push.sh` so that instead of creating a single pipeline
+it creates multiple pipelines. This can be useful when you have a personal CI
+config that you want to run and a separate team / organization pipeline that
+needs to be run as well.
+
+This is not the recommended way however since it would be a lot easier to make
+`cicd/cicd.py` a proper python package instead and put the two configs there
+itself.
+
Contributing
============
diff --git a/jaypore_ci/changelog.py b/jaypore_ci/changelog.py
@@ -1,11 +1,15 @@
from jaypore_ci.config import Version
V = Version.parse
+NEW = "🎁"
+CHANGE = "⚙️"
+BUGFIX = "🐞"
+
version_map = {
V("0.2.29"): {
"changes": [
(
- "Bugfix: When gitea token does not have enough scope log"
+ f"{BUGFIX}: When gitea token does not have enough scope log"
" correctly and exit"
)
],
@@ -14,7 +18,7 @@ version_map = {
V("0.2.28"): {
"changes": [
(
- "Bugfix: When there are multiple (push) remotes, Jaypore CI"
+ f"{BUGFIX}: When there are multiple (push) remotes, Jaypore CI"
" will pick the first one and use that."
)
],
@@ -22,15 +26,19 @@ version_map = {
},
V("0.2.27"): {
"changes": [
- "Jobs older than 1 week will be removed before starting a new pipeline."
+ (
+ f"{NEW}: Jobs older than 1 week will be removed before starting"
+ " a new pipeline."
+ )
],
"instructions": [],
},
V("0.2.26"): {
"changes": [
(
- "The Dockerfile inside `cicd/Dockerfile` now requires a build arg "
- "that specifies the version of Jaypore CI to install."
+ f"{CHANGE}: The Dockerfile inside `cicd/Dockerfile` now"
+ " requires a build arg that specifies the version of Jaypore CI"
+ " to install."
),
],
"instructions": [
@@ -40,10 +48,16 @@ version_map = {
V("0.2.25"): {
"changes": [
(
- "A dockerfile is now used to send context of the codebase to "
- "the docker daemon instead of directly mounting the code."
+ f"{NEW}: A dockerfile is now used to send context of the"
+ " codebase to the docker daemon instead of directly mounting the"
+ " code. This allows us to easily use remote systems for jobs"
)
],
"instructions": [],
},
}
+assert all(
+ line.startswith(NEW) or line.startswith(CHANGE) or line.startswith(BUGFIX)
+ for log in version_map.values()
+ for line in log["changes"]
+), "All change lines must start with one of NEW/CHANGE/BUGFIX"
diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py
@@ -83,22 +83,27 @@ class Job: # pylint: disable=too-many-instance-attributes
It is never created manually. The correct way to create a job is to use
:meth:`~jaypore_ci.jci.Pipeline.job`.
- :param name: The name for the job. Names must be unique across jobs and stages.
- :param command: The command that we need to run for the job. It can be set
- to `None` when `is_service` is True.
- :param is_service: Is this job a service or not? Service jobs are assumed
- to be :class:`~jaypore_ci.interfaces.Status.PASSED` as long as they start.
- They are shut down when the entire pipeline has finished executing.
- :param pipeline: The pipeline this job is associated with.
- :param status: The :class:`~jaypore_ci.interfaces.Status` of this job.
- :param image: What docker image to use for this job.
- :param timeout: Defines how long a job is allowed to run before being
- killed and marked as class:`~jaypore_ci.interfaces.Status.FAILED`.
- :param env: A dictionary of environment variables to pass to the docker run command.
- :param children: Defines which jobs depend on this job's output status.
- :param parents: Defines which jobs need to pass before this job can be run.
- :param stage: What stage the job belongs to. This stage name must exist so
- that we can assign jobs to it.
+ :param name: The name for the job. Names must be unique across jobs
+ and stages.
+ :param command: The command that we need to run for the job. It can be
+ set to `None` when `is_service` is True.
+ :param is_service: Is this job a service or not? Service jobs are assumed
+ to be :class:`~jaypore_ci.interfaces.Status.PASSED` as
+ long as they start. They are shut down when the entire
+ pipeline has finished executing.
+ :param pipeline: The pipeline this job is associated with.
+ :param status: The :class:`~jaypore_ci.interfaces.Status` of this job.
+ :param image: What docker image to use for this job.
+ :param timeout: Defines how long a job is allowed to run before being
+ killed and marked as
+ class:`~jaypore_ci.interfaces.Status.FAILED`.
+ :param env: A dictionary of environment variables to pass to the
+ docker run command.
+ :param children: Defines which jobs depend on this job's output status.
+ :param parents: Defines which jobs need to pass before this job can be
+ run.
+ :param stage: What stage the job belongs to. This stage name must
+ exist so that we can assign jobs to it.
"""
def __init__(
@@ -249,14 +254,20 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
"""
A pipeline acts as a controlling/organizing mechanism for multiple jobs.
- :param repo : Provides information about the codebase.
- :param reporter : Provides reports based on the state of the pipeline.
- :param remote : Allows us to publish reports to somewhere like gitea/email.
- :param executor : Runs the specified jobs.
- :param poll_interval: Defines how frequently (in seconds) to check the
- pipeline status and publish a report.
+ :param repo: Provides information about the codebase.
+ :param reporter: Provides reports based on the state of the pipeline.
+ :param remote: Allows us to publish reports to somewhere like gitea/email.
+ :param executor: Runs the specified jobs.
+ :param poll_interval: Defines how frequently (in seconds) to check the
+ pipeline status and publish a report.
"""
+ # We need a way to avoid actually running the examples. Something like a
+ # "dry-run" option so that only the building of the config is done and it's
+ # never actually run. It might be a good idea to make this an actual config
+ # variable but I'm not sure if we should do that or not.
+ __run_on_exit__ = True
+
def __init__( # pylint: disable=too-many-arguments
self,
*,
@@ -324,9 +335,10 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.run()
- self.executor.teardown()
- self.remote.teardown()
+ if Pipeline.__run_on_exit__:
+ self.run()
+ self.executor.teardown()
+ self.remote.teardown()
return False
def get_status(self) -> Status:
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,4 +1,5 @@
import os
+from pathlib import Path
import unittest
import pytest
@@ -34,6 +35,14 @@ def factory(*, repo, remote, executor, reporter):
return build
+def set_env_keys():
+ os.environ["JAYPORE_GITEA_TOKEN"] = "fake_gitea_token"
+ os.environ["JAYPORE_GITHUB_TOKEN"] = "fake_github_token"
+ os.environ["JAYPORE_EMAIL_ADDR"] = "fake@email.com"
+ os.environ["JAYPORE_EMAIL_PASSWORD"] = "fake_email_password"
+ os.environ["JAYPORE_EMAIL_TO"] = "fake.to@mymailmail.com"
+
+
@pytest.fixture(
scope="function",
params=list(
@@ -53,11 +62,7 @@ def factory(*, repo, remote, executor, reporter):
ids=idfn,
)
def pipeline(request):
- os.environ["JAYPORE_GITEA_TOKEN"] = "fake_gitea_token"
- os.environ["JAYPORE_GITHUB_TOKEN"] = "fake_github_token"
- os.environ["JAYPORE_EMAIL_ADDR"] = "fake@email.com"
- os.environ["JAYPORE_EMAIL_PASSWORD"] = "fake_email_password"
- os.environ["JAYPORE_EMAIL_TO"] = "fake.to@mymailmail.com"
+ set_env_keys()
builder = factory(
repo=request.param["repo"],
remote=request.param["remote"],
@@ -73,3 +78,13 @@ def pipeline(request):
yield builder
else:
yield builder
+
+
+@pytest.fixture(
+ scope="function",
+ params=list((Path(__name__) / "../docs/source/examples").resolve().glob("*.py")),
+ ids=str,
+)
+def doc_example_filepath(request):
+ set_env_keys()
+ yield request.param
diff --git a/tests/test_doc_examples.py b/tests/test_doc_examples.py
@@ -0,0 +1,9 @@
+from jaypore_ci.jci import Pipeline
+
+
+def test_doc_examples(doc_example_filepath):
+ with open(doc_example_filepath, "r", encoding="utf-8") as fl:
+ code = fl.read()
+ Pipeline.__run_on_exit__ = False
+ exec(code) # pylint: disable=exec-used
+ Pipeline.__run_on_exit__ = True
