commit 432512f33d41b84d0ecd4a987c8ad721ab07ab5f
parent 7c06e22ea43e40558741b0e68460063d47c5d110
Author: arjoonn <arjoonn@noreply.localhost>
Date: Fri, 23 Dec 2022 06:19:07 +0000
secrets (!10)
Branch auto created by JayporeCI
<details>
<summary>JayporeCi: 🟢 03fe80b8b2</summary>
```mermaid
flowchart TB
subgraph Pipeline
direction TB
end
subgraph Docker
direction TB
s_Docker(( )) -.-> Docker_0(JciEnv):::passed
s_Docker(( )) -.-> Docker_1(Jci):::passed
end
subgraph Jobs
direction TB
s_Jobs(( )) -.-> Jobs_0(black):::passed
s_Jobs(( )) -.-> Jobs_1(pylint):::passed
s_Jobs(( )) -.-> Jobs_2(pytest):::passed
end
subgraph Publish
direction TB
s_Publish(( )) -.-> Publish_0(DockerHubJci):::passed
s_Publish(( )) -.-> Publish_1(PublishDocs):::passed
s_Publish(( )) -.-> Publish_2(DockerHubJcienv):::passed
end
Pipeline ---> Docker
Docker ---> Jobs
Jobs ---> Publish
classDef pending fill:#aaa, color:black, stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
classDef skipped fill:#aaa, color:black, stroke:black,stroke-width:2px;
classDef assigned fill:#ddd, color:black, stroke:black,stroke-width:2px;
classDef running fill:#bae1ff,color:black,stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
classDef passed fill:#88d8b0, color:black, stroke:black;
classDef failed fill:#ff6f69, color:black, stroke:black;
classDef timeout fill:#ffda9e, color:black, stroke:black;
```
Co-authored-by: arjoonn sharma <arjoonn@midpathsoftware.com>
Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/10
Diffstat:
20 files changed, 183 insertions(+), 62 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
*.pyc
+*.env
+*.age
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
from python:3.11 as jcibase
workdir /app
run apt-get update
-run apt-get install ca-certificates curl gnupg lsb-release -y
+run apt-get install ca-certificates curl zip gnupg lsb-release vim -y
run mkdir -p /etc/apt/keyrings
run curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
run echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
@@ -15,7 +15,12 @@ add pyproject.toml .
add poetry.lock .
run poetry export --with dev > req.txt
run python3 -m pip install -r req.txt
-env PYTHONPATH=/jaypore_ci/run/
+env PYTHONPATH=/jaypore_ci/run/:/app
+env PATH=/jaypore_ci/run/:/app:$PATH
+run wget -O /bin/sops https://github.com/mozilla/sops/releases/download/v3.7.3/sops-v3.7.3.linux
+run wget -O ./age.tar.gz https://github.com/FiloSottile/age/releases/download/v1.0.0/age-v1.0.0-linux-amd64.tar.gz
+run tar xf ./age.tar.gz && mv ./age/age /bin && mv ./age/age-keygen /bin && rm -rf ./age
+run chmod u+x /bin/sops /bin/age /bin/age-keygen
from jcienv as jci
add jaypore_ci/ /app/jaypore_ci
diff --git a/README.md b/README.md
@@ -1,5 +1,6 @@
# Jaypore CI
+
> A CI system that sounds ancient and powerful.
> Like the city of Jaypore.
diff --git a/cicd/build_and_publish_docs.sh b/cicd/build_and_publish_docs.sh
@@ -0,0 +1,27 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+build() {
+ echo "Building docs"
+ sphinx-build docs/source/ docs/build
+ (cd docs/build && zip -r ../../website.zip ./)
+}
+publish() {
+ echo "Publishing docs"
+ source cicd/set_env.sh
+ curl -H "Content-Type: application/zip" \
+ -H "Authorization: Bearer $NETLIFY_TOKEN" \
+ --data-binary "@website.zip" \
+ https://api.netlify.com/api/v1/sites/$NETLIFY_SITEID/deploys
+}
+
+(build)
+if [ $1 == "main" ]
+then
+ (publish)
+else
+ echo "Not publishing since branch is: $1"
+fi
diff --git a/cicd/build_and_push_docker.sh b/cicd/build_and_push_docker.sh
@@ -0,0 +1,11 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source cicd/set_env.sh
+docker login -u arjoonn -p=$DOCKER_PWD
+docker build -t $1:latest .
+docker tag $1:latest arjoonn/$1:latest
+docker push arjoonn/$1:latest
diff --git a/cicd/cicd.py b/cicd/cicd.py
@@ -1,3 +1,4 @@
+import requests
from jaypore_ci import jci
@@ -11,29 +12,6 @@ with jci.Pipeline() as p:
p.job("pylint", "python3 -m pylint jaypore_ci/ tests/")
p.job("pytest", "python3 -m pytest tests/")
with p.stage("Publish", image=jcienv):
- # docs
- p.job("BuildDocs", "sphinx-build -b html docs/source/ docs/build/html")
- p.job("PublishDocs", "bash -c 'echo hi'", depends_on=["BuildDocs"])
- # pypi
- p.job("PoetryBuild", "poetry build")
- p.job("PoetryPublish", "poetry publish", depends_on=["PoetryBuild"])
- # jcienv
- p.job(
- "DockerTagJcienv",
- "docker tag -t jcienv:{p.remote.sha} arjoonn/jcienv:{p.remote.sha}",
- )
- p.job(
- "DockerPublishJcienv",
- "docker push arjoonn/jcienv:{p.remote.sha}",
- depends_on=["DockerTagJcienv"],
- )
- # jci
- p.job(
- "DockerTagJci",
- "docker tag -t jci:{p.remote.sha} arjoonn/jci:{p.remote.sha}",
- )
- p.job(
- "DockerPublishJci",
- "docker push arjoonn/jci:{p.remote.sha}",
- depends_on=["DockerTagJci"],
- )
+ p.job("PublishDocs", f"bash cicd/build_and_publish_docs.sh {p.remote.branch}")
+ p.job("DockerHubJcienv", "bash cicd/build_and_push_docker.sh jcienv")
+ p.job("DockerHubJci", "bash cicd/build_and_push_docker.sh jci")
diff --git a/cicd/edit_secrets.sh b/cicd/edit_secrets.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+main (){
+ NAME=$1
+ SOPS_AGE_KEY_FILE=secrets/$NAME.age sops --decrypt --input-type dotenv --output-type dotenv secrets/$NAME.enc > secrets/$NAME.env
+ vim secrets/$NAME.env
+ sops --encrypt --age $(age-keygen -y secrets/$NAME.age) secrets/$NAME.env > secrets/$NAME.enc
+ rm secrets/$NAME.env
+}
+
+help_message (){
+
+ echo "
+ Easily edit env files.
+ Make sure you have age keys available in
+
+ secrets/<envname>.age
+
+ If that is available you can run the following to edit env files.
+
+ edit_secrets.sh <envname>
+
+ Upon exiting the editor the file will be re-encrypted.
+ "
+}
+
+if [[ $1 == "--help" || $1 == "-h" ]]; then
+ help_message
+ exit 0
+fi
+if [ -z $1 ]; then
+ help_message
+ exit 0
+fi
+(main $1)
diff --git a/cicd/pre-push.githook b/cicd/pre-push.githook
@@ -26,7 +26,12 @@ main() {
-v /tmp/jaypore_$SHA:/jaypore_ci/run \
--workdir /jaypore_ci/run \
jcienv \
- bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run && cd /jaypore_ci/run/ && git clean -fdx && python cicd/cicd.py'
+ bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run \
+ && cd /jaypore_ci/run/ \
+ && git clean -fdx \
+ && rm -r /jaypore_ci/run/secrets \
+ && cp -r /jaypore_ci/repo/secrets/ /jaypore_ci/run \
+ && python cicd/cicd.py'
echo '----------------------------------------------'
}
(main)
diff --git a/cicd/set_env.sh b/cicd/set_env.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+export $(SOPS_AGE_KEY_FILE=secrets/jaypore_ci.age sops --decrypt --input-type dotenv --output-type dotenv secrets/jaypore_ci.enc | xargs)
diff --git a/docs/source/_static/logo.png b/docs/source/_static/logo.png
Binary files differ.
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -22,5 +22,17 @@ exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+html_sidebars = {
+ "**": [
+ "about.html",
+ "navigation.html",
+ "relations.html",
+ "searchbox.html",
+ "donate.html",
+ ]
+}
html_theme = "alabaster"
html_static_path = ["_static"]
+html_theme_options = {
+ "logo": "logo.png",
+}
diff --git a/docs/source/example.png b/docs/source/example.png
Binary files differ.
diff --git a/docs/source/examples.rst b/docs/source/examples.rst
@@ -24,7 +24,6 @@ You can cache your environment dependencies in docker easily.
Complex dependencies between jobs
---------------------------------
-
- A pipeline can have stages.
- Stages are executed one after the other.
- Jobs inside a stage are all run in parallel
diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst
@@ -5,14 +5,16 @@ Installation
------------
-To use Jaypore CI, first install it using the bash script for installation:
+To use **Jaypore CI**, first install it using a bash script.
.. code-block:: console
- $ curl https://raw.githubusercontent.com/theSage21/jaypore_ci/main/setup.sh | bash
+ $ curl \
+ https://raw.githubusercontent.com/theSage21/jaypore_ci/main/setup.sh \
+ | bash
-This will
+Doing this will:
1. Create a directory called `cicd` in the root of your repo.
2. Create a file `cicd/pre-push.githook`
@@ -20,28 +22,24 @@ This will
4. Update your repo's pre-push git hook so that it runs the `cicd/pre-push.githook` file when you push.
-How it works
+Basic config
------------
-1. Every time you run `git push`, the `cicd/pre-push.githook` is run.
- 1. This hook runs your `cicd/cicd.py` file using the `arjoonn/jaypore_ci:latest` docker container.
- 2. The run has access to your local docker instance and so can launch other containers.
-
-
-A Basic Config
---------------
-
-Your entire config is inside `cicd/cicd.py`. Edit it to whatever you like! A simple config would look like this:
+Your entire config is inside `cicd/cicd.py`. Edit it to whatever you like! A basic config would look like this:
.. code-block:: python
from jaypore_ci import jci
with jci.Pipeline(image='mydocker/image') as p:
- p.job("Black", "python3 -m black --check .")
- p.job("Pylint", "python3 -m pylint mycode/ tests/")
- p.job("PyTest", "python3 -m pytest tests/")
+ p.job("Black", "black --check .")
+ p.job("Pylint", "pylint mycode/ tests/")
+ p.job("PyTest", "pytest tests/")
+
+
+After you make these changes you can `git add -Av` and `git commit -m 'added Jaypore CI'`.
+When you do a `git push origin`, that's when the CI system will get triggered and will run the CI.
This config will run three jobs in parallel, using the `mydocker/image` docker image.
diff --git a/docs/source/ideas.rst b/docs/source/ideas.rst
@@ -6,5 +6,25 @@ I'm tired of
- Spending hours figuring out how to do stuff in YAML configs.
- Shift to something else when pricing changes.
-- Debugging why stuff is not working in a CI system but works otherwise.
+- Debugging why stuff is not working only in the CI system.
- Not being able to run CI without internet.
+- In the case of self CI runners hosted on laptops, I don't want my laptop bogged down by jobs that other people pushed.
+
+
+What I like about existing systems
+----------------------------------
+
+- Use docker to run things.
+- Stateless job runs.
+- Job graphs showing status of the run.
+- We cannot merge PRs unless the pipeline passes.
+- Able to handle multiple languages easily in the same project.
+
+
+Concepts used
+-------------
+
+- We use a git hook as a trigger mechanism. There is no possibility that some "CI server" is down.
+- Jobs are run on the machine that pushed the job by default. If you write bad code, your machine suffers first.
+- CI run status is posted directly in the PR description. You don't have to click and reach another website to see what your job is doing.
+- All jobs run inside docker containers.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -6,19 +6,29 @@
Welcome to Jaypore CI's documentation!
======================================
-**Jaypore CI** is a Python library for continuous integration / testing / delivery.
+**Jaypore CI** is a small system for continuous integration / testing / delivery.
-It is slightly different from things like github actions, gitlab CI, drone CI and so on.
+It is different from the usual suspects like github actions, gitlab CI, drone CI and so on.
- The configuration language is python.
- CI runs on your local machine by default.
-- There is no "server". CI reports are added to your pull request / merge request description.
+- There is no "server". You can run offline.
For example, here's a CI pipeline for a project.
-.. image:: example.png
- :alt: Example pipeline graph
+.. code-block:: python
+
+ from jaypore_ci import jci
+
+ with jci.Pipeline(image='mydockerhub/env_image') as p:
+ p.job("Black", "black --check .")
+ p.job("Pylint", "pylint mycode/ tests/")
+ p.job("PyTest", "pytest tests/")
+
+
+Go through the :doc:`getting_started` doc to set up your first instance.
+
Contents
---------------
diff --git a/jaypore_ci/gitea.py b/jaypore_ci/gitea.py
@@ -122,9 +122,9 @@ class Gitea(Remote): # pylint: disable=too-many-instance-attributes
"""
Will publish the report to the remote.
- report: Report to write to remote.
- status: One of ["pending", "success", "error", "failure", "warning"]
- This is the dot next to each commit in gitea.
+ :param report: Report to write to remote.
+ :param status: One of ["pending", "success", "error", "failure",
+ "warning"] This is the dot next to each commit in gitea.
"""
assert status in ("pending", "success", "error", "failure", "warning")
issue_id = self.get_pr_id()
diff --git a/jaypore_ci/interfaces.py b/jaypore_ci/interfaces.py
@@ -8,7 +8,6 @@ respectively.
class TriggerFailed(Exception):
"Failure to trigger a job"
- ...
class Executor:
diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py
@@ -107,11 +107,9 @@ class Job: # pylint: disable=too-many-instance-attributes
def update_report(self):
"""
- Update the status report.
- Usually called when a job changes some of it's internal state like:
- - logs are updated
- - status has changed
- - last_check is called
+ Update the status report. Usually called when a job changes some of
+ it's internal state like when logs are updated or when status has
+ changed.
"""
self.logging().debug("Update report")
status = {
@@ -304,7 +302,7 @@ class Pipeline: # pylint: disable=too-many-instance-attributes
</details>"""
- def __render_graph__(self) -> str:
+ def __render_graph__(self) -> str: # pylint: disable=too-many-locals
"""
Render a mermaid graph given the jobs in the pipeline.
"""
diff --git a/secrets/jaypore_ci.enc b/secrets/jaypore_ci.enc
@@ -0,0 +1,10 @@
+DOCKER_USER=ENC[AES256_GCM,data:kodwp9BekQ==,iv:4kxwikp9m8obe3Hb1MCLdDOGga1VUnqxLPU4E6s25D4=,tag:XaRuNYq8k9QNlQ/Ftq4Atg==,type:str]
+DOCKER_PWD=ENC[AES256_GCM,data:GViemE/cvlOfMdmeDdqaM83J5E5fs590KIftj5Ul3QBafTQ+,iv:bIizRqtgXS4IY7Ml01etoQ01UvV5Efkicz2epTTlcNI=,tag:DHkC4pOimYL3siMgAbkyRg==,type:str]
+NETLIFY_TOKEN=ENC[AES256_GCM,data:h1yMdRv4ps+hstSCiLN388g/vq6VwYIsXT8C0Uxq7n5bA04syWosivoj1g==,iv:+m20LBp2NqzQ/EOdTXRLTe76N5kdMTqgGcuZ9QsCiL4=,tag:b7Kew1KQ6PfCIWZLvVF/9Q==,type:str]
+NETLIFY_SITEID=ENC[AES256_GCM,data:lr2RcnMdHKsXTWfwhx5VM0n9hiCF2MaezajRJRjYtmD8ILFY,iv:cHyNXTYZ7nW+JAvp29jncf6GWmCHN0obJvihoYQYHic=,tag:1618mQjs7CMFfwhLvaVQ0Q==,type:str]
+sops_mac=ENC[AES256_GCM,data:FIMFiy2ZQHmi/8WS+4etoiQhEFgXoGlPIUY14icexxxj2GdkGth3suDLFdQ5x92Jnq5quBkz52qQmE56fO/coIYoBCR2rZzaEmu68gDRlsfUv48vMTIHueAvN/DaEiyFwpGcg4HWoZNirbciCDyI3HBK8/C4/IpWb/arwrrNIKY=,iv:InPgDMxTdZigJN/T3TsqvQoUuBMYalAovP2dT5a7Fx8=,tag:3agYFIaLQTtr3QOPzDNdww==,type:str]
+sops_version=3.7.3
+sops_age__list_0__map_recipient=age1u0zd477nnqqausg4vtxl3laxz73t5tgt9qw8enmpeeadcmes3eusw3v3m9
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvSzl4T2NhRFl3d2xDZkxE\nUk5lMlBtemV5UXVGMkxCbE1ma3JBbDJWcWdnCmlRejRqbUFJTnVPWVF2dUh1TThk\nZnVvWjBjTXNHTUpRd2NxU2pXVkVqQzgKLS0tIFcyNmhkbE5SazQvZ01nZ2dlcklX\ndVh5NkFuZkx3enV2Y3hFMUxmMzJhUVkKy6VXe+51BaDRN8I0KsyaviseaHSCInMY\nxMxiKM8zcs39OAv7VAkQZ6O6lhBi0digoqYMOYxPykzE7U9MfTE2aA==\n-----END AGE ENCRYPTED FILE-----\n
+sops_lastmodified=2022-12-23T06:01:23Z
+sops_unencrypted_suffix=_unencrypted
