commit c3ed2e98006f0a6a735edbd3750e63b2fc61c38c
parent 1b581a9256eac7b3c4dc2d2f15c45f451e9445c9
Author: arjoonn <arjoonn@noreply.localhost>
Date: Sun, 25 Dec 2022 04:46:10 +0000
secrets_docs (!15)
Branch auto created by JayporeCI
<details>
<summary>JayporeCi: 🟢 86456c1785</summary>
```mermaid
flowchart TB
subgraph Pipeline
direction TB
end
subgraph Docker
direction TB
s_Docker(( )) -.-> Docker_0(JciEnv):::passed
s_Docker(( )) -.-> Docker_1(Jci):::passed
end
subgraph Jobs
direction TB
s_Jobs(( )) -.-> Jobs_0(pytest):::passed
s_Jobs(( )) -.-> Jobs_1(PublishDocs):::passed
s_Jobs(( )) -.-> Jobs_2(pylint):::passed
s_Jobs(( )) -.-> Jobs_3(black):::passed
end
subgraph Publish
direction TB
s_Publish(( )) -.-> Publish_0(DockerHubJcienv):::passed
s_Publish(( )) -.-> Publish_1(DockerHubJci):::passed
end
Pipeline ---> Docker
Docker ---> Jobs
Jobs ---> Publish
classDef pending fill:#aaa, color:black, stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
classDef skipped fill:#aaa, color:black, stroke:black,stroke-width:2px;
classDef assigned fill:#ddd, color:black, stroke:black,stroke-width:2px;
classDef running fill:#bae1ff,color:black,stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
classDef passed fill:#88d8b0, color:black, stroke:black;
classDef failed fill:#ff6f69, color:black, stroke:black;
classDef timeout fill:#ffda9e, color:black, stroke:black;
```
Co-authored-by: arjoonn sharma <arjoonn@midpathsoftware.com>
Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/15
Diffstat:
11 files changed, 90 insertions(+), 66 deletions(-)
diff --git a/README.md b/README.md
@@ -1,5 +1,6 @@
# Jaypore CI
+
Documentation is at : https://www.jayporeci.in
## Usage
diff --git a/cicd/build_and_publish_docs.sh b/cicd/build_and_publish_docs.sh
@@ -11,7 +11,6 @@ build() {
}
publish() {
echo "Publishing docs"
- source cicd/set_env.sh
curl -H "Content-Type: application/zip" \
-H "Authorization: Bearer $NETLIFY_TOKEN" \
--data-binary "@website.zip" \
diff --git a/cicd/build_and_push_docker.sh b/cicd/build_and_push_docker.sh
@@ -4,7 +4,6 @@ set -o errexit
set -o nounset
set -o pipefail
-source cicd/set_env.sh
docker login -u arjoonn -p=$DOCKER_PWD
docker build --target $1 -t $1:latest .
docker tag $1:latest arjoonn/$1:latest
diff --git a/cicd/cicd.py b/cicd/cicd.py
@@ -7,10 +7,10 @@ with jci.Pipeline() as p:
p.job("JciEnv", f"docker build --target jcienv -t jcienv:{p.remote.sha} .")
p.job("Jci", f"docker build --target jci -t jci:{p.remote.sha} .")
with p.stage("Jobs", image=jcienv):
+ p.job("PublishDocs", f"bash cicd/build_and_publish_docs.sh {p.remote.branch}")
p.job("black", "python3 -m black --check .")
p.job("pylint", "python3 -m pylint jaypore_ci/ tests/")
p.job("pytest", "python3 -m pytest tests/")
with p.stage("Publish", image=jcienv):
- p.job("PublishDocs", f"bash cicd/build_and_publish_docs.sh {p.remote.branch}")
p.job("DockerHubJcienv", "bash cicd/build_and_push_docker.sh jcienv")
p.job("DockerHubJci", "bash cicd/build_and_push_docker.sh jci")
diff --git a/cicd/pre-push.githook b/cicd/pre-push.githook
@@ -1,37 +0,0 @@
-#! /bin/bash
-#
-set -o errexit
-set -o nounset
-set -o pipefail
-
-
-main() {
- SHA=$(git rev-parse HEAD)
- REPO_ROOT=$(git rev-parse --show-toplevel)
- TOKEN=$(echo "url=$(git remote -v|grep push|awk '{print $2}')"|git credential fill|grep password|awk -F= '{print $2}')
- # We will mount the current dir into /jaypore_ci/repo
- # Then we will copy things over to /jaypore_ci/run
- # Then we will run git clean to remove anything that is not in git
- # Then we call the actual cicd code
- #
- # We also pass docker.sock to the run so that jaypore_ci can create docker containers
- echo '----------------------------------------------'
- echo "JayporeCi: "
- JAYPORE_GITEA_TOKEN="${JAYPORE_GITEA_TOKEN:-$TOKEN}" docker run \
- -d \
- --name jaypore_ci_$SHA \
- -e JAYPORE_GITEA_TOKEN \
- -v /var/run/docker.sock:/var/run/docker.sock \
- -v $REPO_ROOT:/jaypore_ci/repo:ro \
- -v /tmp/jaypore_$SHA:/jaypore_ci/run \
- --workdir /jaypore_ci/run \
- jcienv \
- bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run \
- && cd /jaypore_ci/run/ \
- && git clean -fdx \
- && rm -r /jaypore_ci/run/secrets \
- && cp -r /jaypore_ci/repo/secrets/ /jaypore_ci/run \
- && python cicd/cicd.py'
- echo '----------------------------------------------'
-}
-(main)
diff --git a/cicd/pre-push.sh b/cicd/pre-push.sh
@@ -0,0 +1,47 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+run() {
+ export SECRETS_PATH=secrets
+ export SECRETS_FILENAME=jaypore_ci
+ export $(SOPS_AGE_KEY_FILE=/jaypore_ci/repo/$SECRETS_PATH/$SECRETS_FILENAME.age sops --decrypt --input-type dotenv --output-type dotenv /jaypore_ci/repo/$SECRETS_PATH/$SECRETS_FILENAME.enc | xargs)
+ cp -r /jaypore_ci/repo/. /jaypore_ci/run
+ cd /jaypore_ci/run/
+ git clean -fdx
+ # Change the name of the file if this is not cicd.py
+ python /jaypore_ci/run/$JAYPORE_CODE_DIR/cicd.py
+}
+
+
+hook() {
+ SHA=$(git rev-parse HEAD)
+ REPO_ROOT=$(git rev-parse --show-toplevel)
+ TOKEN=$(echo "url=$(git remote -v|grep push|awk '{print $2}')"|git credential fill|grep password|awk -F= '{print $2}')
+ JAYPORE_CODE_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ JAYPORE_CODE_DIR=$(basename $JAYPORE_CODE_DIR)
+ # We will mount the current dir into /jaypore_ci/repo
+ # Then we will copy things over to /jaypore_ci/run
+ # Then we will run git clean to remove anything that is not in git
+ # Then we call the actual cicd code
+ #
+ # We also pass docker.sock to the run so that jaypore_ci can create docker containers
+ echo '----------------------------------------------'
+ echo "JayporeCi: "
+ JAYPORE_GITEA_TOKEN="${JAYPORE_GITEA_TOKEN:-$TOKEN}" docker run \
+ -d \
+ --name jaypore_ci_$SHA \
+ -e JAYPORE_GITEA_TOKEN \
+ -e JAYPORE_CODE_DIR=$JAYPORE_CODE_DIR \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -v $REPO_ROOT:/jaypore_ci/repo:ro \
+ -v /tmp/jaypore_$SHA:/jaypore_ci/run \
+ --workdir /jaypore_ci/run \
+ arjoonn/jci:latest \
+ bash -c "bash /jaypore_ci/repo/$JAYPORE_CODE_DIR/pre-push.sh run"
+ echo '----------------------------------------------'
+}
+("$@")
diff --git a/docs/source/examples.rst b/docs/source/examples.rst
@@ -111,7 +111,7 @@ Running on cloud/remote machine
- Since the executor is docker:
- We can get the remote machine's docker socket by using [ssh socket forwarding](https://medium.com/@dperny/forwarding-the-docker-socket-over-ssh-e6567cfab160)
- - Then we can set Jaypore CI to use the remote docker socket by editing `cicd/pre-push.githook`
+ - Then we can set Jaypore CI to use the remote docker socket by editing `cicd/pre-push.sh`
- Now all jobs will run on the remote machine.
diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst
@@ -9,17 +9,15 @@ To use **Jaypore CI**, first install it using a bash script.
.. code-block:: console
- $ curl \
- https://raw.githubusercontent.com/theSage21/jaypore_ci/main/setup.sh \
- | bash
+ $ curl https://get.jayporeci.in | bash
Doing this will:
1. Create a directory called `cicd` in the root of your repo.
-2. Create a file `cicd/pre-push.githook`
+2. Create a file `cicd/pre-push.sh`
3. Create a file `cicd/cicd.py`
-4. Update your repo's pre-push git hook so that it runs the `cicd/pre-push.githook` file when you push.
+4. Update your repo's pre-push git hook so that it runs the `cicd/pre-push.sh` file when you push.
Basic config
diff --git a/docs/source/ideas.rst b/docs/source/ideas.rst
@@ -28,3 +28,7 @@ Concepts used
- Jobs are run on the machine that pushed the job by default. If you write bad code, your machine suffers first.
- CI run status is posted directly in the PR description. You don't have to click and reach another website to see what your job is doing.
- All jobs run inside docker containers.
+- Use `SOPS <https://github.com/mozilla/sops>`_ to manage secrets during CI.
+ - `.gitignore` your encryption key files.
+ - Commit your secrets in an encrypted form.
+ - Update your `pre-push.sh` file to inject the secrets into your CI environment.
diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py
@@ -2,6 +2,7 @@
The code submodule for Jaypore CI.
"""
import time
+import os
import re
from enum import Enum
from itertools import product
@@ -185,7 +186,7 @@ class Job: # pylint: disable=too-many-instance-attributes
Gets the environment variables for a given job by interpolating it with
the pipeline's environment.
"""
- return {**self.pipeline.pipe_kwargs.get("env", {}), **self.env}
+ return {**os.environ, **self.pipeline.pipe_kwargs.get("env", {}), **self.env}
class Pipeline: # pylint: disable=too-many-instance-attributes
diff --git a/setup.sh b/setup.sh
@@ -7,35 +7,46 @@ main (){
LOCAL_HOOK=$(echo $REPO_ROOT/.git/hooks/pre-push)
IMAGE='arjoonn/jci:latest'
echo "Working in repo: $REPO_ROOT"
+ echo "Adding git hook at: $LOCAL_HOOK"
+ CICD_ROOT=cicd
mkdir $REPO_ROOT/cicd || echo 'Moving on..'
cat > $REPO_ROOT/cicd/cicd.py << EOF
from jaypore_ci import jci
-with jci.Pipeline(
- image="$IMAGE", # NOTE: Change this to whatever you need
- timeout=15 * 60
-) as p:
- p.in_parallel(
- p.job("pwd", name="Pwd"),
- p.job("tree", name="Tree"),
- p.job("python3 -m black --check .", name="Black"),
- p.job("python3 -m pylint jaypore_ci/ tests/", name="PyLint"),
- p.job("python3 -m pytest tests/", name="PyTest"),
- ).should_pass()
+with jci.Pipeline() as p:
+ p.job("Workingdir", "pwd")
+ p.job("Tree", "tree")
+ p.job("Black", "black --check .")
+ p.job("PyLint", "pylint jaypore_ci/ tests/")
+ p.job("PyTest", "pytest tests/")
EOF
- cat > $REPO_ROOT/cicd/pre-push.githook << EOF
+ cat > $REPO_ROOT/cicd/pre-push.sh << EOF
#! /bin/bash
-#
+
set -o errexit
set -o nounset
set -o pipefail
-main() {
+run() {
+ export SECRETS_PATH=secrets
+ export SECRETS_FILENAME=jaypore_ci
+ export \$(SOPS_AGE_KEY_FILE=/jaypore_ci/repo/\$SECRETS_PATH/\$SECRETS_FILENAME.age sops --decrypt --input-type dotenv --output-type dotenv /jaypore_ci/repo/\$SECRETS_PATH/\$SECRETS_FILENAME.enc | xargs)
+ cp -r /jaypore_ci/repo/. /jaypore_ci/run
+ cd /jaypore_ci/run/
+ git clean -fdx
+ # Change the name of the file if this is not cicd.py
+ python /jaypore_ci/run/\$JAYPORE_CODE_DIR/cicd.py
+}
+
+
+hook() {
SHA=\$(git rev-parse HEAD)
REPO_ROOT=\$(git rev-parse --show-toplevel)
TOKEN=\$(echo "url=\$(git remote -v|grep push|awk '{print \$2}')"|git credential fill|grep password|awk -F= '{print \$2}')
+ JAYPORE_CODE_DIR=\$( cd -- "\$( dirname -- "\${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ JAYPORE_CODE_DIR=\$(basename \$JAYPORE_CODE_DIR)
# We will mount the current dir into /jaypore_ci/repo
# Then we will copy things over to /jaypore_ci/run
# Then we will run git clean to remove anything that is not in git
@@ -48,18 +59,19 @@ main() {
-d \\
--name jaypore_ci_\$SHA \\
-e JAYPORE_GITEA_TOKEN \\
+ -e JAYPORE_CODE_DIR=\$JAYPORE_CODE_DIR \\
-v /var/run/docker.sock:/var/run/docker.sock \\
-v \$REPO_ROOT:/jaypore_ci/repo:ro \\
-v /tmp/jaypore_\$SHA:/jaypore_ci/run \\
--workdir /jaypore_ci/run \\
$IMAGE \\
- bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run && cd /jaypore_ci/run/ && git clean -fdx && python cicd/cicd.py'
+ bash -c "bash /jaypore_ci/repo/\$JAYPORE_CODE_DIR/pre-push.sh run"
echo '----------------------------------------------'
}
-(main)
+("\$@")
EOF
echo "Creating git hook for pre-commit"
- chmod u+x $REPO_ROOT/cicd/pre-push.githook
+ chmod u+x $REPO_ROOT/cicd/pre-push.sh
if test -f "$LOCAL_HOOK"; then
if test -f "$LOCAL_HOOK.local"; then
@@ -67,7 +79,7 @@ EOF
echo $LOCAL_HOOK
echo $LOCAL_HOOK.local
echo "Please link"
- echo " Jaypore hook : $REPO_ROOT/cicd/pre-push.githook"
+ echo " Jaypore hook : $REPO_ROOT/cicd/pre-push.sh"
echo "with"
echo " Existing hook: $LOCAL_HOOK"
echo "manually by editing the existing hook file"
@@ -80,7 +92,7 @@ EOF
echo "$REPO_ROOT/.git/hooks/pre-push.local" >> $REPO_ROOT/.git/hooks/pre-push
fi
fi
- echo "$REPO_ROOT/cicd/pre-push.githook" >> $REPO_ROOT/.git/hooks/pre-push
+ echo "$REPO_ROOT/cicd/pre-push.sh" >> $REPO_ROOT/.git/hooks/pre-push
chmod u+x $LOCAL_HOOK
}
