diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..5d29e4d09169e7a68278007eb96c999693595ea2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git/ +frontend/node_modules/ +backend/out +frontend/.gradle +.gradle/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1c57d561a75d41f8216027708dae6e6505675ae..1a90de7239154f4472704d8e9d73c69503eb519f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ --- stages: + - pre-build - test - build - deploy-beta @@ -20,7 +21,9 @@ variables: APP_NAME: faidare JAR_PATH: "backend/build/libs/${APP_NAME}.jar" GIT_DEPTH: 0 - + IMAGE_TAG: $CI_COMMIT_REF_SLUG + ELASTIC_VERSION: "6.6.2" + DOCKER_OPTS: "--mtu=1450" # Gradle cache for all jobs cache: @@ -30,6 +33,31 @@ cache: - "frontend/.gradle/" - "frontend/node_modules/" +# PRE-BUILD + +build-loader-docker-image: + image: registry.forgemia.inra.fr/urgi-is/docker-rare/docker-git:latest + stage: pre-build + services: + - docker:20.10.6-dind + script: + # build the image + - docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG} . + - docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest . + # Login before pushing the image + - docker login registry.forgemia.inra.fr -u $CONTAINER_REGISTRY_USERNAME -p $CONTAINER_REGISTRY_TOKEN + # push the built image + - docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG} + # only push latest tag on master branch + - if [ "master" == "${CI_COMMIT_REF_SLUG}" ] ; then docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest ; fi; + only: + changes: + - scripts/* + - backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json + - backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json + - .gitlab-ci.yml + allow_failure: true + # TESTS @@ -39,6 +67,13 @@ lint: tags: - openstack script: "./gradlew lint" + cache: + key: "$CI_COMMIT_REF_NAME" + policy: pull + paths: + - ".gradle" + - "frontend/.gradle/" + - "frontend/node_modules/" test-and-sonarqube: @@ -52,7 +87,7 @@ test-and-sonarqube: # because we need to pass some variables, but they are passed to _all_ containers # so they fail the start of other docker images like urgi/docker-browsers # the only solution is to override the entrypoint of the service and pass the arguments manually - - name: docker.elastic.co/elasticsearch/elasticsearch:6.5.4 + - name: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} alias: elasticsearch # discovery.type=single-node # single-node is necessary to start in development mode @@ -65,11 +100,15 @@ test-and-sonarqube: SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache GIT_DEPTH: "0" # Tells git to fetch all the branches of the project, required by the analysis task cache: - key: "${CI_JOB_NAME}" + key: "${CI_COMMIT_REF_NAME}" + policy: pull-push paths: + - ".gradle" + - "frontend/.gradle/" + - "frontend/node_modules/" - .sonar/cache script: - - ./gradlew :frontend:test --parallel + - ./gradlew :frontend:assemble --parallel - ./gradlew :backend:test jacocoTestReport --parallel - find /tmp/node/*/bin -name node -exec ln -s {} /tmp/node/node \; - export PATH="/tmp/node/:$PATH" @@ -78,7 +117,7 @@ test-and-sonarqube: reports: junit: - ./backend/build/test-results/test/TEST-*.xml - - ./frontend/karma-junit-tests-report/TEST*.xml + # - ./frontend/karma-junit-tests-report/TEST*.xml only: refs: - merge_requests @@ -100,6 +139,13 @@ build: stage: build script: - ./gradlew assemble + cache: + key: "${CI_COMMIT_REF_NAME}" + policy: pull + paths: + - ".gradle" + - "frontend/.gradle/" + - "frontend/node_modules/" artifacts: paths: - "$JAR_PATH" @@ -134,7 +180,7 @@ restart-config-server-openstack: # DEPLOY -.deploy-to-vm-proxmox: &deploy_to_vm_proxmox +.deploy-to-vm: &deploy_to_vm # Hidden job which serves as template for executed jobs below. # See https://docs.gitlab.com/ee/ci/yaml/#anchors retry: 2 @@ -155,38 +201,16 @@ restart-config-server-openstack: - .gitlab-ci.yml - backend/src/**/* - frontend/**/* - - -.deploy-to-vm-openstack: &deploy_to_vm_openstack - # Hidden job which serves as template for executed jobs below. - # See https://docs.gitlab.com/ee/ci/yaml/#anchors - retry: 2 - tags: - - openstack - script: - ## SSH initialization - - eval $(ssh-agent -s) - - ssh-add <(echo "${SSH_PRIVATE_KEY}") - - ssh -o StrictHostKeyChecking=no ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} 'echo "Successfully connected on $(hostname)"' - # Copy jar - - scp ./backend/build/libs/${APP_NAME}.jar ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK}:/tmp/${APP_NAME}-${ENV}.jar - - ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo mv /tmp/${APP_NAME}-${ENV}.jar /opt/bootapp/${APP_NAME}-${ENV}.jar ; sudo chown -R bootapp:bootapp /opt/bootapp/" - # Restarting service with the updated jar and the according Spring profiles enabled - - ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo systemctl restart bootapp@${APP_NAME}-${ENV}" - - eval $(ssh-agent -k) - - echo "Deploy done. Application should be available at http://${SERVER_IP_OPENSTACK}:${APP_PORT}/${CONTEXT_PATH}" - only: - changes: - - .gitlab-ci.yml - - backend/src/**/* - - frontend/**/* - when: manual allow_failure: false deploy-to-beta: + tags: + - openstack stage: deploy-beta - extends: .deploy-to-vm-openstack + extends: .deploy-to-vm variables: + SERVER_USER: ${SERVER_USER_OPENSTACK} + SERVER_IP: ${SERVER_IP_OPENSTACK_DEV} APP_PORT: ${BETA_FAIDARE_PORT} ENV: beta CONTEXT_PATH: faidare-beta @@ -199,9 +223,13 @@ deploy-to-beta: when: always deploy-to-staging: + tags: + - openstack stage: deploy-staging - extends: .deploy-to-vm-openstack + extends: .deploy-to-vm variables: + SERVER_USER: ${SERVER_USER_OPENSTACK} + SERVER_IP: ${SERVER_IP_OPENSTACK_DEV} APP_PORT: ${STAGING_FAIDARE_PORT} ENV: staging CONTEXT_PATH: faidare-staging @@ -214,9 +242,13 @@ deploy-to-staging: when: manual deploy-to-int: + tags: + - proxmox stage: deploy-production - extends: .deploy-to-vm-proxmox + extends: .deploy-to-vm variables: + SERVER_USER: ${SERVER_USER_PROXMOX} + SERVER_IP: ${SERVER_IP_PROXMOX} APP_PORT: ${INT_FAIDARE_PORT} ENV: int CONTEXT_PATH: faidare-int @@ -226,9 +258,13 @@ deploy-to-int: when: manual deploy-to-prod-public: + tags: + - proxmox stage: deploy-production - extends: .deploy-to-vm-proxmox + extends: .deploy-to-vm variables: + SERVER_USER: ${SERVER_USER_PROXMOX} + SERVER_IP: ${SERVER_IP_PROXMOX} APP_PORT: ${PROD_PUBLIC_FAIDARE_PORT} ENV: prod-public CONTEXT_PATH: faidare @@ -238,9 +274,13 @@ deploy-to-prod-public: when: manual deploy-to-prod-private: + tags: + - proxmox stage: deploy-production - extends: .deploy-to-vm-proxmox + extends: .deploy-to-vm variables: + SERVER_USER: ${SERVER_USER_PROXMOX} + SERVER_IP: ${SERVER_IP_PROXMOX} APP_PORT: ${PROD_PRIVATE_FAIDARE_PORT} ENV: prod-private CONTEXT_PATH: faidare-private diff --git a/.secrets.baseline b/.secrets.baseline index 78cadc12aeee2d976a218886f8b9702855af0cb8..58672400dda7618e0fea2e3886828061441b2b73 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "frontend/package-lock.json|^.secrets.baseline$", "lines": null }, - "generated_at": "2021-04-16T10:41:53Z", + "generated_at": "2021-08-02T15:23:45Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -51,21 +51,21 @@ "hashed_secret": "2907dcd1b70a82032e52be9b6b804abbb4a7525e", "is_secret": false, "is_verified": false, - "line_number": 83, + "line_number": 127, "type": "Base64 High Entropy String" }, { "hashed_secret": "dd447c7c799dd4ebaacca8f0ad3da45a097d7211", "is_secret": false, "is_verified": false, - "line_number": 199, + "line_number": 228, "type": "Base64 High Entropy String" }, { "hashed_secret": "8074db38f8a8acec1a147bc5daf2799ff6693fff", "is_secret": false, "is_verified": false, - "line_number": 214, + "line_number": 247, "type": "Base64 High Entropy String" } ], diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d9b28b366813c26130f7cfbbc39bd3bf9eb3e172 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine +LABEL Author="Raphaël FLORES <raphael.flores@inrae.fr>" + +COPY scripts/harvest.sh /opt/scripts/ + +# COPY dao settings +COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json + +# COPY dao mappings +COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/ + +RUN apk add --update --no-cache bash curl jq parallel wget grep gzip sed date coreutils + +RUN chmod +x /opt/scripts/harvest.sh +RUN mkdir ~/.parallel && touch ~/.parallel/will-cite + +ENTRYPOINT ["/opt/scripts/harvest.sh"] diff --git a/README.md b/README.md index 20f005d61ee952e9fb69184bc31fbf5e6213a210..028a48c73e8654e923592c7be5a691cf9ffb35ca 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,38 @@ cd frontend ng serve ``` +## Harvest + +Before all, take care to get data locally before running any indexing script. + +### TL;DR + +Data indexing to your local Elasticsearch is done using the following command (take care to change the path to local data). Note that your local Elasticsearch instance should be already runing using `docker-compose up`: + +```sh +docker run -t --volume /path/to/local/data:/opt/data/ --network=container:elasticsearch-faidare registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest -jsonDir /opt/data/ --help +``` +Remove the `--help` parameter to run the loading with default params. + +### Portability + +#### Docker + +[TL;DR](#TLDR) section above expects to have an available docker image on the forgemia docker registry. The Gitlab CI rebuil it when needed, but you can update or push such an image using the following commands: + +```sh +# build the image +docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest . + +# Login before pushing the image +docker login registry.forgemia.inra.fr/urgi-is/docker-rare -u <your ForgeMIA username> + +# push the built image +docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest +``` + +That should ease the indexing of data without having to craft a dedicated environment. + ## GitLab CI When creating merge requests on the ForgeMIA GitLab, the GitLab CI will diff --git a/backend/build.gradle.kts b/backend/build.gradle.kts index 22b76b8318103b68f93f2cd3791428e2dd88b66e..9ff140544f08a54151ddf9ca56e164de76e37a35 100644 --- a/backend/build.gradle.kts +++ b/backend/build.gradle.kts @@ -100,8 +100,8 @@ dependencies { implementation("org.springframework.cloud:spring-cloud-starter-config") // Elasticsearch - implementation("org.elasticsearch:elasticsearch:6.5.4") - implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.5.4") + implementation("org.elasticsearch:elasticsearch:6.6.2") + implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.6.2") // Swagger implementation("io.swagger:swagger-annotations:1.5.21") diff --git a/docker-compose.yml b/docker-compose.yml index e06ad4c5dd76b14668f6dc7e79fe1067baf4f1a1..fd9fbf1fe581e4a17e00bea4a3548ba0a2e316c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4 + image: docker.elastic.co/elasticsearch/elasticsearch:6.6.2 container_name: elasticsearch-faidare environment: - discovery.type=single-node @@ -9,7 +9,7 @@ services: - 9200:9200 kibana: - image: docker.elastic.co/kibana/kibana:6.5.4 + image: docker.elastic.co/kibana/kibana:6.6.2 container_name: kibana-faidare environment: - "ELASTICSEARCH_URL=http://elasticsearch:9200" diff --git a/frontend/src/karma.conf.js b/frontend/src/karma.conf.js index ccbec7f955d961c3bacdd27743e4b61e32450136..495fcc2c4cc1304b8c4e861f656c7cb857764c7b 100644 --- a/frontend/src/karma.conf.js +++ b/frontend/src/karma.conf.js @@ -44,6 +44,8 @@ module.exports = function (config) { flags: ['--no-sandbox'] // required for Gitlab CI } }, + captureTimeout: 1200000, // increase timeouts: https://github.com/jasmine/jasmine/issues/1413#issuecomment-334247097 + browserNoActivityTimeout: 120000, singleRun: false, restartOnFileChange: true });