.gitlab-ci.yml

workflow:
  rules:
    # Execute on tags, MR or protected branches
    - if: $CI_COMMIT_TAG
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # Avoid duplicated pipelines in MR context
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_REF_PROTECTED == "true"
    # Ignore pipeline for filthy commits
    - if: $CI_COMMIT_MESSAGE =~ /^wip.*/i
      when: never
  auto_cancel:
    on_new_commit: interruptible

stages:
  - Build
  - Static Analysis
  - Documentation
  - Test
  - Applications Test
  - Ship

variables:
  OTBTF_VERSION: 4.4.0-rc1
  OTBTF_SRC: /src/otbtf  # OTBTF source directory path in image
  DATADIR: $CI_PROJECT_DIR/test/data
  CACHE_IMAGE_CPU: $CI_REGISTRY_IMAGE:build-cache-cpu
  CACHE_IMAGE_GPU: $CI_REGISTRY_IMAGE:build-cache-gpu
  BRANCH_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
  DOCKER_DRIVER: overlay2
  DOCKER_BUILDKIT: 1
  BUILDX_BUILDER: container
  BUILDX_NO_DEFAULT_ATTESTATIONS: 1

default:
  tags: [ godzilla ]
  interruptible: true
  image:
    name: $BRANCH_IMAGE
    pull_policy: always

.docker_build_base:
  image: docker:27.3.1
  services:
    - name: docker:27.3.1-dind
  before_script:
    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - docker buildx create --name=container --driver=docker-container --use --bootstrap
  after_script:
    - docker buildx rm --keep-state container

docker_image:
  stage: Build
  only:
    - merge_requests
    - develop
  extends: .docker_build_base
  script:
    - >
      docker buildx build --push -t $BRANCH_IMAGE
      --cache-from type=registry,ref=$CACHE_IMAGE_CPU
      --cache-to type=registry,ref=$CACHE_IMAGE_CPU,mode=max
      --build-arg DEV_IMAGE=true
      "."

.static_analysis_base:
  stage: Static Analysis
  allow_failure: true
  only:
    - merge_requests

flake8:
  extends: .static_analysis_base
  script:
    - flake8 $OTBTF_SRC/otbtf --exclude=tensorflow_v1x

pylint:
  extends: .static_analysis_base
  script:
    - pylint $OTBTF_SRC/otbtf --ignore=tensorflow_v1x

codespell:
  rules:
  extends: .static_analysis_base
  script:
    - codespell otbtf doc README.md

cppcheck:
  extends: .static_analysis_base
  script:
    - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction .

docs:
  stage: Documentation
  tags: [ stable ]
  needs: []
  image: python:3.10-slim
  variables:
    PTH: public_test
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - if: $CI_COMMIT_REF_NAME == /master/
      variables:
        PTH: public
    - changes:
      - doc/**/*
      - "*.{md,txt}"
      - mkdocs.yml
      - .readthedocs.yaml
  before_script:
    - pip install -r doc/doc_requirements.txt
  script:
    - mkdocs build --site-dir $PTH
  artifacts:
    paths:
      - public
      - public_test

.tests_base:
  only:
    - merge_requests
  artifacts:
    reports:
      junit: report_*.xml
    expire_in: 1 week
    when: on_failure

ctest:
  stage: Test
  extends: .tests_base
  needs: ["docker_image"]
  variables:
    CTEST_OUTPUT_ON_FAILURE: 1
    OTB_TEST_UNITS: "Tensorflow|PanSharpening|Projection|Transform|IOGDAL"
    OTB_BUILD: /src/otb/build/OTB/build
  script:
    - cd /src/otb/otb && git lfs pull
    - ln -s $CI_PROJECT_DIR/test/data $OTBTF_SRC/test/data
    - ln -s $CI_PROJECT_DIR/test/models $OTBTF_SRC/test/models
    - cd $OTB_BUILD
    - ctest -L $OTB_TEST_UNITS --output-junit $CI_PROJECT_DIR/report_ctest.xml
  after_script:
    - cp -r $OTB_BUILD/Testing/Temporary/* $CI_PROJECT_DIR/artifacts_ctest
  artifacts:
    paths:
      - $CI_PROJECT_DIR/artifacts_ctest

python_api:
  stage: Test
  extends: .tests_base
  variables:
    API_TEST_TMP: /tmp/api_tests_tmp
  script:
    - mkdir $API_TEST_TMP
    - TMPDIR=$API_TEST_TMP python -m pytest -svv --junitxml=report_api.xml test/api_unittest.py
  after_script:
    - saved_model_cli show --dir $API_TEST_TMP/model_from_pimg --all
    - cp -r $API_TEST_TMP $CI_PROJECT_DIR/artifacts_test_api
  artifacts:
    paths:
      - $CI_PROJECT_DIR/artifacts_test_api

.applications_test_base:
  stage: Applications Test
  extends: .tests_base
  needs: ["ctest", "python_api"]
  #when: manual

crc_book:
  extends: .applications_test_base
  allow_failure: true
  variables:
    CRC_BOOK_TMP: /tmp/crc_book_tests_tmp
  script:
    - mkdir -p $CRC_BOOK_TMP
    - TMPDIR=$CRC_BOOK_TMP python -m pytest -v --junitxml=report_tutorial.xml test/tutorial_unittest.py
  after_script:
    - cp -r $CRC_BOOK_TMP $CI_PROJECT_DIR/artifacts_crc_book
  artifacts:
    paths:
      - $CI_PROJECT_DIR/artifacts_crc_book

decloud:
  extends: .applications_test_base
  allow_failure: true
  variables:
    DATASET_DECLOUD: https://nextcloud.inrae.fr/s/aNTWLcH28zNomqk/download
  script:
    - git clone https://github.com/CNES/decloud.git
    - pip install -r $PWD/decloud/docker/requirements.txt
    - wget -q $DATASET_DECLOUD -O file.zip && unzip file.zip
    - export DECLOUD_DATA_DIR="$PWD/decloud_data"
    - pytest -v decloud/tests/train_from_tfrecords_unittest.py

sr4rs:
  extends: .applications_test_base
  variables:
    DATASET_S2: https://nextcloud.inrae.fr/s/EZL2JN7SZyDK8Cf/download/sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
    DATASET_SR4RS: https://nextcloud.inrae.fr/s/kDms9JrRMQE2Q5z/download
  script:
    - wget -qO sr4rs_sentinel2_bands4328_france2020_savedmodel.zip $DATASET_S2
    - unzip -o sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
    - wget -qO sr4rs_data.zip $DATASET_SR4RS
    - unzip -o sr4rs_data.zip
    - rm -rf sr4rs
    - git clone https://github.com/remicres/sr4rs.git
    - export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs
    - python -m pytest -v --junitxml=report_sr4rs.xml test/sr4rs_unittest.py

geos_enabled:
  extends: .applications_test_base
  script:
    - python -m pytest -v --junitxml=report_geos_enabled.xml test/geos_test.py

planetary_computer:
  extends: .applications_test_base
  script:
    - pip install pystac_client planetary_computer
    - python -m pytest -v --junitxml=report_pc_enabled.xml test/pc_test.py

numpy_gdal_otb:
  extends: .applications_test_base
  script:
    - python -m pytest -v --junitxml=report_numpy.xml test/numpy_test.py

rio:
  extends: .applications_test_base
  script:
    - pip install rasterio --no-binary rasterio
    - python -m pytest -v --junitxml=report_rio.xml test/rio_test.py

nodata:
  extends: .applications_test_base
  script:
    - python -m pytest -v --junitxml=report_nodata.xml test/nodata_test.py

.ship_base:
  extends: .docker_build_base
  only:
    - tags
  variables:
    DOCKERHUB_BASE: mdl4eo/otbtf
    CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
    DOCKERHUB_IMAGE_BASE: $DOCKERHUB_BASE:$OTBTF_VERSION

deploy_cpu:
  stage: Ship
  extends: .ship_base
  variables:
    IMAGE_CPU: $CI_REGISTRY_PUBIMG-cpu
    IMAGE_CPUDEV: $CI_REGISTRY_PUBIMG-cpu-dev
    DOCKERHUB_CPU: $DOCKERHUB_IMAGE_BASE-cpu
    DOCKERHUB_CPUDEV: $DOCKERHUB_IMAGE_BASE-cpu-dev
    DOCKERHUB_LATEST: $DOCKERHUB_BASE:latest
  script:
    # cpu
    - >
      docker buildx build --push -t $IMAGE_CPU
      --cache-from type=registry,ref=$CACHE_IMAGE_CPU
      .
    # cpu-dev
    - >
      docker buildx build --push -t $IMAGE_CPUDEV
      --cache-from type=registry,ref=$CACHE_IMAGE_CPU
      --build-arg DEV_IMAGE=true
      .
    # push images on dockerhub
    - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
    - docker tag $IMAGE_CPU $DOCKERHUB_CPU && docker push $DOCKERHUB_CPU
    - docker tag $IMAGE_CPUDEV $DOCKERHUB_CPUDEV && docker push $DOCKERHUB_CPUDEV
    # latest = cpu image
    - docker tag $IMAGE_CPU $DOCKERHUB_LATEST && docker push $DOCKERHUB_LATEST

deploy_gpu:
  stage: Ship
  extends: .ship_base
  variables:
    IMAGE_GPU: $CI_REGISTRY_PUBIMG-gpu
    IMAGE_GPUDEV: $CI_REGISTRY_PUBIMG-gpu-dev
    DOCKERHUB_GPU: $DOCKERHUB_IMAGE_BASE-gpu
    DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
    DOCKERHUB_GPULATEST: $DOCKERHUB_BASE:latest-gpu
  script:
    # gpu
    - >
      docker buildx build --push -t $IMAGE_GPU
      --cache-from type=registry,ref=$CACHE_IMAGE_GPU
      --cache-from type=registry,ref=$CACHE_IMAGE_CPU
      --cache-to type=registry,ref=$CACHE_IMAGE_GPU,mode=max
      --build-arg WITH_CUDA=true
      .
    # gpu-dev
    - >
      docker buildx build --push -t $IMAGE_GPUDEV
      --cache-from type=registry,ref=$CACHE_IMAGE_GPU
      --cache-from type=registry,ref=$CACHE_IMAGE_CPU
      --build-arg WITH_CUDA=true --build-arg DEV_IMAGE=true
      .
    # push gpu-* images on dockerhub
    - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
    - docker tag $IMAGE_GPU $DOCKERHUB_GPU && docker push $DOCKERHUB_GPU
    - docker tag $IMAGE_GPUDEV $DOCKERHUB_GPUDEV && docker push $DOCKERHUB_GPUDEV
    # latest-gpu = gpu image
    - docker tag $IMAGE_GPU $DOCKERHUB_GPULATEST && docker push $DOCKERHUB_GPULATEST