Fix makefile

Signed-off-by: Yuanmao Zhu <[email protected]>
hyperledger-cello · Feb 18, 2023 · e0f5775 · e0f5775
1 parent 6717c47
commit e0f5775
Show file tree

Hide file tree

Showing 9 changed files with 330 additions and 185 deletions.
diff --git a/Makefile b/Makefile
@@ -5,12 +5,23 @@
 #
 # -------------------------------------------------------------
 # This makefile defines the following targets, feel free to run "make help" to see help info
-#
-#   - clean:          Cleans the build area
-#   - dockerhub-pull: Pulling service images from dockerhub
-#   - reset:          Clean up and remove local storage (only use for development)
+# 	- all (default): Builds all targets and runs all tests/checks
+#   - license:		  Checks sourrce files for Apache license header
+#   - check:          Setup as master node, and runs all tests/checks, will be triggered by CI
+#   - help:           Output the help instructions for each command
+#   - doc:       	  Start a local web service to explore the documentation
+#   - docker[-clean]: Build/clean docker images locally
 #   - start:          Start the cello service
 #   - stop:           Stop the cello service, and remove all service containers
+#   - restart:        Stop the cello service and then start
+#   - setup-master:   Setup the host as a master node, install pkg and download docker images
+#   - setup-worker:   Setup the host as a worker node, install pkg and download docker images
+#   - clean:          Cleans the docker containers.
+#   - deep-clean: 	  Clean up all docker images and local storage.
+#   - docker-compose: Start development docker-compose.
+#   - local: 		  Run all services ad-hoc
+#   - reset:          Clean up and remove local storage (only use for development)
+
 
 GREEN  := $(shell tput -Txterm setaf 2)
 WHITE  := $(shell tput -Txterm setaf 7)
@@ -99,9 +110,21 @@ else
 	export DEBUG?=True
 endif
 
-docker-clean: stop image-clean ##@Clean all existing images
-
-license:
+HELP_FUN = \
+	%help; \
+	while(<>) { push @{$$help{$$2 // 'options'}}, [$$1, $$3] if /^([a-zA-Z\-]+)\s*:.*\#\#(?:@([a-zA-Z\-]+))?\s(.*)$$/ }; \
+	print "usage: make [target]\n\n"; \
+	for (sort keys %help) { \
+		print "${WHITE}$$_:${RESET}\n"; \
+		for (@{$$help{$$_}}) { \
+			$$sep = " " x (32 - length $$_->[0]); \
+			print "  ${YELLOW}$$_->[0]${RESET}$$sep${GREEN}$$_->[1]${RESET}\n"; \
+	}; \
+	print "\n"; }
+
+all: check
+
+license:  ##@Code Check source files for Apache license header
 	scripts/check_license.sh
 
 check: ##@Code Check code format
@@ -115,90 +138,114 @@ check: ##@Code Check code format
 	MODE=dev make stop
 	make check-dashboard
 
-check-dashboard:
-	docker compose -f tests/dashboard/docker-compose.yml up --abort-on-container-exit || (echo "check dashboard failed $$?"; exit 1)
+doc: ##@Documentation Build local online documentation and start serve
+	command -v mkdocs >/dev/null 2>&1 || pip install -r docs/requirements.txt || pip3 -r docs/requirements.txt
+	mkdocs serve -f mkdocs.yml
 
-clean:
-	make remove-docker-compose
+help: ##@Help Show this help.
+	@perl -e '$(HELP_FUN)' $(MAKEFILE_LIST)
 
-deep-clean:
+docker: images ##@Build Build all required docker images locally
+
+docker-clean:##@Clean Clean docker images locally  
+	make stop
+	make clean-images
+
+start: ##@Service Start service
+	make start-docker-compose
+
+stop: ##@Service Stop service
+	if [ "$(CONFIG_DOCKER_COMPOSE_DEPLOY)" = "y" ]; then \
+		make stop-docker-compose; \
+	else \
+		make stop-k8s; \
+	fi
+
+restart: stop start ##@Service Restart services
+
+setup-master: ##@Environment Setup dependency for master node
+	cd scripts/master_node && bash setup.sh
+
+setup-worker: ##@Environment Setup dependency for worker node
+	cd scripts/worker_node && bash setup.sh $(WORKER_TYPE)
+
+clean: ##@Clean Stop services and clean docker containers.
+	make stop
+	if docker ps -a | grep "cello-"; then \
+		docker ps -a | grep "cello-" | awk '{print $1}' | xargs docker rm -f >/dev/null 2>&1; \
+	fi
+
+deep-clean: ##@Clean Stop services, clean docker images and remove mounted local storage.
+	make stop
 	make clean
-	make remove-hyperledger-fabric-containers
-	make image-clean
+	make clean-docker-images
 	rm -rf $(LOCAL_STORAGE_PATH)
 
-image-clean: clean ##@Clean all existing images to rebuild
+docker-compose:##@Development Start development docker-compose
+	api-engine fabric docker-rest-agent dashboard
+
+reset:##@Development clean up and remove local storage (only use for development)
+	make clean 
+	echo "Clean up and remove all local storage..."
+	rm -rf ${LOCAL_STORAGE_PATH}/*
+
+local:##@Development Run all services ad-hoc
+	make docker-compose start-docker-compose 
+
+## Help rules
+clean-images: 
+	make clean
 	echo "Clean all cello related images, may need to remove all containers before"
 	docker images | grep "cello-" | awk '{print $3}' | xargs docker rmi -f
 
+check-dashboard:
+	docker compose -f tests/dashboard/docker-compose.yml up --abort-on-container-exit || (echo "check dashboard failed $$?"; exit 1)
+
 start-docker-compose:
 	docker compose -f bootup/docker-compose-files/${COMPOSE_FILE} up -d --force-recreate --remove-orphans
 
-start: ##@Service Start service
-	make start-docker-compose
-
 stop-docker-compose:
 	echo "Stop all services with bootup/docker-compose-files/${COMPOSE_FILE}..."
 	docker compose -f bootup/docker-compose-files/${COMPOSE_FILE} stop
 	echo "Stop all services successfully"
 
-remove-docker-compose:
-	make stop-docker-compose
-	echo "Remove all services with bootup/docker-compose-files/${COMPOSE_FILE}..."
-	if docker ps -a | grep "cello-"; then \
-		docker ps -a | grep "cello-" | awk '{print $1}' | xargs docker rm -f >/dev/null 2>&1; \
-		rm -rf /opt/cello; \
-	fi
-	echo "Remove all services successfully"
-
-remove-hyperledger-fabric-containers:
-	echo "Remove all nodes ..."
-	if docker ps -a | grep "hyperledger-fabric"; then \
-		docker ps -a | grep "hyperledger-fabric" | awk '{print $1}' | xargs docker rm -f >/dev/null 2>&1; \
-		rm -rf /opt/hyperledger; \
-	fi
-	echo "Remove all nodes successfully"
-
-stop: ##@Service Stop service
-	if [ "$(CONFIG_DOCKER_COMPOSE_DEPLOY)" = "y" ]; then \
-		make stop-docker-compose; \
-	else \
-		make stop-k8s; \
-	fi
-
-restart: stop start ##@Service Restart service
+images: api-engine docker-rest-agent fabric dashboard
 
-api-engine: # for debug only now
-	docker build -t hyperledger/cello-api-engine:latest -f build_image/docker/common/api-engine/Dockerfile.in ./ --platform linux/$(ARCH); \
+api-engine: 
+	docker build -t hyperledger/cello-api-engine:latest -f build_image/docker/common/api-engine/Dockerfile.in ./ --platform linux/$(ARCH)
 
-docker-rest-agent: # for debug only now
-	docker build -t hyperledger/cello-agent-docker:latest -f build_image/docker/agent/docker-rest-agent/Dockerfile.in ./ --build-arg pip=$(PIP) --platform linux/$(ARCH); \
+docker-rest-agent:
+	docker build -t hyperledger/cello-agent-docker:latest -f build_image/docker/agent/docker-rest-agent/Dockerfile.in ./ --build-arg pip=$(PIP) --platform linux/$(ARCH)
 
 fabric:
 	docker image pull yeasy/hyperledger-fabric:2.2.0
 
-dashboard: # for debug only now
+dashboard:
 	docker build -t hyperledger/cello-dashboard:latest -f build_image/docker/common/dashboard/Dockerfile.in ./
 
-docker-compose: api-engine fabric docker-rest-agent dashboard
 
-local: docker-compose start-docker-compose 
 
 
 .PHONY: \
-	deep-clean \
-	docker-clean \
+	all \
 	license \
-	stop-docker-compose \
-	remove-docker-compose \
-	remove-hyperledger-fabric-containers \
-	restart \
+	check \
+	doc \ 
+	help \
+	docker \
+	docker-clean \
 	start \
+	stop \
+	restart \
+	clean \
+	deep-clean \
 	api-engine \
 	fabric \
-	docker-rest-agent \
 	dashboard \
 	docker-compose \
+	reset \
 	local \
-	check \ 
-	check-dashboard \
+	clean-images \
+	start-docker-compose \
+	stop-docker-compose \
+	images \
diff --git a/bootup/docker-compose-files/docker-compose.yml b/bootup/docker-compose-files/docker-compose.yml
@@ -4,7 +4,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# Local `/opt/cello/mongo` will be used for the db storage.
+# Local `/opt/cello` will be used for the db storage.
 
 #  cello-nginx: proxy to access operator dashboard service, listen on 8080
 #  cello-operator-dashboard: dashboard service for operators

diff --git a/docs/concepts.md b/docs/concepts.md
@@ -10,22 +10,23 @@ Api engine supply the core function, all operations through the api service.
 
 Host path mappings:
 
-* /opt/cello/api-engine/media:/var/www/media {>>store all media files<<}
-* /var/run/docker.sock:/var/run/docker.sock {>>Used for agent containers launch, which will deploy fabric, eg. network<<}
-* (==optional==) $ROOT_PATH/src/api-engine:/var/www/server {>>When run in debug mode, MODE=dev, will mapping the source code into container, ROOT_PATH is the source code path.<<}
+- /opt/cello/api-engine/media:/var/www/media {>>store all media files<<}
+- /var/run/docker.sock:/var/run/docker.sock {>>Used for agent containers launch, which will deploy fabric, eg. network<<}
+- (==optional==) $ROOT_PATH/src/api-engine:/var/www/server {>>When run in debug mode, MODE=dev, will mapping the source code into container, ROOT_PATH is the source code path.<<}
 
 #### 1.1.2 API Engine Tasks
 
 This component run all the async longtime task for api engine, it's receive tasks through redis, and can dynamic set the thread pool.
-    ```
+
+```
 
 #### 1.1.3 Postgres
 
 Store all the data in postgres database, and the storage path is mapping out on the host.
 
 Host path mappings:
 
-* /opt/cello/postgres:/var/lib/postgresql/data {>>Store all db data.<<}
+- /opt/cello/postgres:/var/lib/postgresql/data {>>Store all db data.<<}
 
 #### 1.1.4 Redis
 
@@ -41,8 +42,9 @@ Host path mappings:
 
 ### 2.1 [Kubernetes](agents/kubernetes.md)
 
-### 2.2 [Ansible](agents/ansible.md)
+### 2.2 [Fabric Operator](agents/fabric-operator.md)
 
 [^1]: running containers of cello master service
 
 <a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.
+```
diff --git a/docs/contributing.md b/docs/contributing.md
@@ -0,0 +1,106 @@
+[//]: # "SPDX-License-Identifier: CC-BY-4.0"
+
+Any kind of contribution is encouraged, e.g., [Jira items](https://jira.hyperledger.org/projects/CE/issues) or [patchsets](https://github.com/hyperledger/cello).
+
+## 1. LF ID Application
+
+All the tools require an Linux Foundation (LF) ID.
+
+If you do not have an LF ID, [here is how to get one](https://wiki.hyperledger.org/display/CA/Setting+up+an+LFID).
+
+## 2. Jira board usage
+
+We are using [Jira](https://jira.hyperledger.org/projects/CE) to track the project progress, and welcome to report bug issues or create to-do tasks there. Each item should try keeping simple and focused, hence easy to fix and review.
+
+After login with your LF ID, you can see those task items with one of the following statuses:
+
+- `To Do`: Available for picking and fix.
+- `In Progress`: Some on already picked it (check the assignee) to work on.
+- `Under Review`: Related patchset has been submitted for review, and added as comment under the Jira item.
+- `Done`: Patchset merged, the item has been resolved.
+
+In brief, if you want to contribute, create or find some `To Do` item, and assign it to yourself, then update its status to `In Progress`. After the item is fixed, remember to mark it as `Under Review` and `Done` when the patch is submitted and merged.
+
+## 3. Questions and discussions
+
+- [Chat](https://discord.gg/hyperledger): technical discussions and questions
+
+## 4. Code Commit Steps
+
+The project employs [GitHub](https://github.com/hyperledger/cello) as the code commit/review system.
+
+- Before committing code, please go to [Jira](https://jira.hyperledger.org/projects/CE) to create a new task or check if there's related existing one, then assign yourself as the assignee. Notice each task will get a Jira number like [CE-26](https://jira.hyperledger.org/browse/CE-26).
+
+- Clone the project to your working directory.
+
+```bash
+$ git clone [email protected]:hyperledger/cello.git
+```
+
+(Optionally) Config your git name and email if not setup previously.
+
+```bash
+$ git config user.name "your name"
+$ git config user.email "your email"
+```
+
+- Assign yourself a `To Do` Jira task, mark it as `In progress`, then create a branch with the Jira task number off of your cloned repository, e.g., for CE-26, it can be:
+
+```bash
+$ cd cello
+$ git checkout -b CE-26
+```
+
+- After modifying the code, run `make check` to make sure all the checking is passed. Then Commit your code with `-s` to sign-off, and `-a` to automatically add changes (or run `git add .` to include all changes manually).
+
+```bash
+$ make check
+  ...
+  py27: commands succeeded
+  py30: commands succeeded
+  py35: commands succeeded
+  flake8: commands succeeded
+  congratulations :)
+
+$ git commit -s -a
+```
+
+Example commit msg may look like (take CE-1234 for example):
+
+```bash
+[CE-1234] A short description of your change with no period at the end
+
+You can add more details here in several paragraphs, but please keep each line
+width less than 80 characters. A bug fix should include the issue number.
+
+CE-1234 #done.
+
+Signed-off-by: Your Name <[email protected]>
+```
+
+- Submit your PR using [hub](https://github.com/github/hub/), and mark the
+  corresponding Jira item as `Under Review`.
+
+```bash
+docs % git push ryjones
+Enumerating objects: 7, done.
+Counting objects: 100% (7/7), done.
+Delta compression using up to 4 threads
+Compressing objects: 100% (4/4), done.
+Writing objects: 100% (4/4), 835 bytes | 835.00 KiB/s, done.
+Total 4 (delta 3), reused 0 (delta 0), pack-reused 0
+remote: Resolving deltas: 100% (3/3), completed with 3 local objects.
+To github.com:ryjones/cello.git
+   90f0d2e..ec42e8a  master -> master
+docs % hub pull-request
+https://github.com/hyperledger/cello/pull/145
+docs %
+```
+
+Notice you will get a GitHub PR url like `https://github.com/hyperledger/cello/pull/145`, open it and check the status.
+
+After the ci checking passed, add [reviewers](https://wiki.hyperledger.org/projects/cello#contributors) to the reviewer list and also post the GitHub PR url in the chat channel. The patch will be merged into the `master` branch after passing the review, then mark the Jira item as `Done`.
+
+- If you need to refine the patch further as the reviewers may suggest, you can change on the same branch, and commit the new code with `git commit -a --amend`, and then use the `git review` command again.
+
+<a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,4 +1,3 @@
-mkdocs>=1.0.4
+mkdocs>= 1.4.2
 mkdocs-material>=4.4.0
-pygments>=2.4.2
-pymdown-extensions>=6.0
+pygments>=2.14.0