1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
# This code is licensed from CircleCI to the user under the MIT license.
# See here for details: https://circleci.com/developer/orbs/licensing
version: 2.1
description: |
    Continue a pipeline based on paths of changed files. This can be useful in a monorepo setup where one may want to trigger different workflows based on which module(s) in the repo has changed. This orb does not support server at this time.
display:
    home_url: https://circleci.com/docs/guides/orchestrate/dynamic-config/
    source_url: https://github.com/CircleCI-Public/path-filtering-orb
orbs:
    continuation: circleci/continuation@2.0.1
    jq: circleci/jq@3.0.2
commands:
    generate-config:
        description: |
            Generate config file from the config list.
        parameters:
            config-list-path:
                default: /tmp/filtered-config-list
                description: |
                    A file path to append config paths. Each path in this file should be relative to the working directory.
                type: string
            generated-config-path:
                default: /tmp/generated-config.yml
                description: |
                    A file path for the generated config file.
                type: string
        steps:
            - run:
                command: |
                    #!/usr/bin/env bash

                    # GitHub's URL for the latest release, will redirect.
                    GITHUB_BASE_URL="https://github.com/mikefarah/yq"
                    DESTDIR="${DESTDIR:-/usr/local/bin}"
                    VERSION="4.34.1"

                    function installYq() {
                      echo "Checking For yq + CURL"
                      if command -v curl >/dev/null 2>&1 && ! command -v yq >/dev/null 2>&1; then
                        echo "Installing yq v${VERSION}"

                        uname -a | grep Darwin > /dev/null 2>&1 && OS='darwin' || OS='linux'
                        PLATFORM=$(uname -m)
                        RELEASE_URL="${GITHUB_BASE_URL}/releases/download/v${VERSION}/yq_${OS}_${PLATFORM}"

                        # save the current checkout dir
                        CHECKOUT_DIR=$(pwd)

                        SCRATCH=$(mktemp -d || mktemp -d -t 'tmp')
                        cd "$SCRATCH" || exit

                        wget "${RELEASE_URL}" -O /usr/local/bin/yq &&\
                        chmod +x /usr/local/bin/yq

                        command -v yq >/dev/null 2>&1

                        echo "Installation finished"
                        # Delete the working directory when the install was successful.
                        cd "$CHECKOUT_DIR" || exit
                        rm -r "$SCRATCH"
                        return $?
                      else
                        command -v curl >/dev/null 2>&1 || { echo >&2 "PATH-FILTERING ORB ERROR: CURL is required. Please install."; exit 1; }
                        command -v yq >/dev/null 2>&1 || { echo >&2 "PATH-FILTERING ORB ERROR: yq is required. Please install"; exit 1; }
                        return $?
                      fi
                    }

                    function generateConfig() {
                      echo "Config list ==="

                      cat "${PARAM_CONFIG_LIST_PATH}"

                      echo
                      echo "Generated YAML ==="

                      touch "${PARAM_GENERATED_CONFIG_PATH}"

                      # shellcheck disable=SC2154,SC2016
                      < "${PARAM_CONFIG_LIST_PATH}" \
                      awk 'NF {$1=$1; printf "\"%s\" ", $0}' \
                      | xargs yq eval-all 'explode(.) | . as $item ireduce ({}; . * $item )' \
                      | tee "${PARAM_GENERATED_CONFIG_PATH}"
                    }

                    installYq
                    generateConfig
                environment:
                    PARAM_CONFIG_LIST_PATH: <<parameters.config-list-path>>
                    PARAM_GENERATED_CONFIG_PATH: <<parameters.generated-config-path>>
                name: Generate config
    set-parameters:
        description: |
            Generates a set of pipeline parameters from `mapping` at `output-path`. Python is required to run this command.
        parameters:
            base-revision:
                default: main
                description: |
                    The revision to compare the current one against for the purpose of determining changed files.
                type: string
            config-path:
                default: .circleci/continue_config.yml
                description: |
                    The location of the config to continue the pipeline with, please note that this parameter will be ignored if the user passes the config file per mapping in the mapping parameter
                type: string
            exclude:
                default: ""
                description: |
                    List of paths to exclude from the mapping. One path per line.
                type: string
            mapping:
                default: ""
                description: |
                    Mapping of path regular expressions to pipeline parameters and values. One mapping per line, whitespace-delimited. If duplicate parameter keys are found, the last matching pattern will apply.
                type: string
            output-path:
                default: /tmp/pipeline-parameters.json
                description: |
                    Path to save the generated parameters to.
                type: string
            same-base-run:
                default: true
                description: |
                    This is value is used to know what to do when the base revision is the same as the current commit. When set to true, it will compare with the previous commit. When set to false, it will act as if not changes were done.
                type: boolean
        steps:
            - jq/install
            - run:
                command: "#!/usr/bin/env bash\nMAPPING=\"$(echo \"$MAPPING\" | circleci env subst)\"\nEXCLUDE=\"$(echo \"$EXCLUDE\" | circleci env subst)\"\nOUTPUT_PATH=\"$(echo \"$OUTPUT_PATH\" | circleci env subst)\"\nCONFIG_PATH=\"$(echo \"$CONFIG_PATH\" | circleci env subst)\"\nBASE_REVISION=\"$(echo \"$BASE_REVISION\" | circleci env subst)\"\nSAME_BASE_RUN=\"$(echo \"$SAME_BASE_RUN\" | circleci env subst)\"\n\ndeclare MAPPING_PATH\ndeclare CONFIG_FILE\ndeclare PARAM_NAME\ndeclare PARAM_VALUE\n\nfiltered_config_list_file=\"/tmp/filtered-config-list\"\nfiles_changed_file=\"/tmp/files-changed-list\"\ngit checkout \"$BASE_REVISION\"\ngit checkout \"$CIRCLE_SHA1\"\nMERGE_BASE=$(git merge-base \"$BASE_REVISION\" \"$CIRCLE_SHA1\")\n\nfunction is_mapping_line() {\n    local line=\"$1\"\n    local trimmed\n    trimmed=$(echo \"$line\" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')\n    if [[ -z \"$trimmed\" || \"$trimmed\" == \\#* ]]; then\n        return 1\n    else \n        return 0\n    fi\n}\n\nfunction write_mappings() {\n    local output_path=\"$1\"\n    shift\n    {\n        echo -n \"{\"\n        local first=true\n        for i in \"$@\"; do\n            key=\"${i%% *}\" \n            raw_value=\"${i#* }\"\n            if ! echo \"$raw_value\" | jq -c . >/dev/null 2>&1; then\n                value=$(printf '%s' \"$raw_value\" | jq -R .)\n            else\n                value=\"$raw_value\"\n            fi\n            if [ \"$first\" = true ]; then\n                first=false\n            else\n                echo -n \",\"\n            fi\n            printf '\"%s\":%s' \"$key\" \"$value\"\n        done\n        echo \"}\"\n    } > \"$output_path\"\n}\n\nfunction write_filtered_config_list() {\n    local file_path=\"$1\"\n    shift\n    : > \"$file_path\"\n    for file in \"$@\"; do\n        echo \"$file\" >> \"$file_path\"\n    done\n}\n\nfunction already_in_list() {\n    local item=\"$1\"\n    shift\n    for existing in \"$@\"; do\n        [[ \"$existing\" == \"$item\" ]] && return 0\n    done\n    return 1\n}\n\nif [[ \"$MERGE_BASE\" == \"$CIRCLE_SHA1\" ]]; then\n    if [ \"$SAME_BASE_RUN\" == \"0\" ]; then\n        echo \"Already in the base revision, exiting\"\n        echo \"{}\" > \"$OUTPUT_PATH\"\n        : > \"$filtered_config_list_file\"\n        exit 0\n    fi\n    if git rev-parse HEAD~1; then\n        MERGE_BASE=$(git rev-parse HEAD~1)\n    else\n        # This is the empty tree SHA in case the repo has no commits and the previous command dosn't work\n        MERGE_BASE=\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"\n    fi\nfi\n\necho \"Comparing $MERGE_BASE...$CIRCLE_SHA1\"\nFILES_CHANGED=$(git -c core.quotepath=false diff --name-only \"$MERGE_BASE\" \"$CIRCLE_SHA1\")\n\necho \"$FILES_CHANGED\" > $files_changed_file\n\nif [ -f \"$MAPPING\" ]; then\n    # In this case MAPPING is a file with the mappings\n    MAPPING=$(cat \"$MAPPING\")\nfi\nif [ -f \"$EXCLUDE\" ]; then\n    # In this case EXCLUDE is a file with the exclusions\n    EXCLUDE=$(cat \"$EXCLUDE\")\nfi\n\nfiltered_mapping=()\nfiltered_files_set=()\nwhile IFS= read -r line; do\n    if is_mapping_line \"$line\"; then\n        read -ra tokens \\<<< \"$line\"\n        if [ \"${#tokens[@]}\" -eq 1 ]; then # <path>\n            MAPPING_PATH=\"${tokens[0]}\"\n        elif [ \"${#tokens[@]}\" -eq 2 ]; then # <path> <config>\n            MAPPING_PATH=\"${tokens[0]}\"\n            CONFIG_FILE=\"${tokens[1]}\"\n        elif [ \"${#tokens[@]}\" -eq 3 ]; then # <path> <key_param> <value_param>\n            MAPPING_PATH=\"${tokens[0]}\"\n            PARAM_NAME=\"${tokens[1]}\"\n            PARAM_VALUE=\"${tokens[2]}\"\n        elif [ \"${#tokens[@]}\" -eq 4 ]; then # <path> <key_param> <value_param> <config>\n            MAPPING_PATH=\"${tokens[0]}\"\n            PARAM_NAME=\"${tokens[1]}\"\n            PARAM_VALUE=\"${tokens[2]}\"\n            CONFIG_FILE=\"${tokens[3]}\"\n        else\n            echo \"Invalid mapping length of ${#tokens[@]}\"\n            exit 1\n        fi\n        regex=\"^$MAPPING_PATH\\$\"\n        for i in $FILES_CHANGED; do\n            PATH_EXCLUDED=0\n            if echo \"$i\" | grep -P \"$regex\" >/dev/null 2>&1; then\n                while IFS= read -r ex; do\n                    regex_exclude=\"^$ex\\$\"\n                    if echo \"$i\" | grep -P \"$regex_exclude\" >/dev/null 2>&1; then\n                        PATH_EXCLUDED=1\n                        break\n                    fi\n                done \\<<< \"$EXCLUDE\"\n                if [ \"$PATH_EXCLUDED\" -eq 1 ]; then\n                    continue\n                fi\n\n                if [ -n \"$PARAM_VALUE\" ]; then\n                    filtered_mapping+=(\"$PARAM_NAME $PARAM_VALUE\")\n                fi\n                if [[ -n \"$CONFIG_FILE\" ]] && ! already_in_list \"$CONFIG_FILE\" \"${filtered_files_set[@]}\"; then\n                    filtered_files_set+=(\"$CONFIG_FILE\")\n                fi\n                break\n            fi\n        done\n    fi\ndone \\<<< \"$MAPPING\"\n\nif [[ ${#filtered_mapping[@]} -eq 0 && -n \"$PARAM_VALUE\" ]]; then\n    echo \"No change detected in the paths defined in the mapping parameter\"\nfi\n\nwrite_mappings \"$OUTPUT_PATH\" \"${filtered_mapping[@]}\"\nif [[ ${#filtered_files_set[@]} -eq 0 ]]; then\n    filtered_files_set+=(\"$CONFIG_PATH\")\nfi\nwrite_filtered_config_list \"$filtered_config_list_file\" \"${filtered_files_set[@]}\"\n"
                environment:
                    BASE_REVISION: << parameters.base-revision >>
                    CONFIG_PATH: << parameters.config-path >>
                    EXCLUDE: <<parameters.exclude>>
                    MAPPING: << parameters.mapping >>
                    OUTPUT_PATH: << parameters.output-path >>
                    SAME_BASE_RUN: <<parameters.same-base-run>>
                name: Set parameters
executors:
    default:
        description: |
            This is a sample executor using Docker
        docker:
            - image: cimg/base:<<parameters.tag>>
        parameters:
            tag:
                default: current
                description: |
                    Pick a specific cimg/base image variant: https://hub.docker.com/r/cimg/base/tags
                type: string
jobs:
    filter:
        circleci_ip_ranges: << parameters.circleci_ip_ranges >>
        description: |
            Continues a pipeline in the `setup` state based with static config and a set of pipeline parameters based on the changes in this push.
            The mapping should be a set of items like so: <path regular expression> <pipeline parameter> <value> Multiple mappings can be supplied on separate lines. If the regular expression matches any file changed between HEAD and the base revision, the pipeline parameter will be set to the supplied value for the setup workflow continuation. This way the continuation config can be filtered to only perform relevant tasks.
        executor: <<parameters.executor>>
        parameters:
            base-revision:
                default: main
                description: |
                    The revision to compare the current one against for the purpose of determining changed files.
                type: string
            checkout:
                default: true
                description: Whether to run an optional checkout step before continuing
                type: boolean
            checkout_method:
                default: blobless
                description: Type of checkout to use. Supported are full and blobless. Default to blobless.
                enum:
                    - full
                    - blobless
                type: enum
            circleci_domain:
                default: circleci.com
                description: The domain of the CircleCI installation - defaults to circleci.com. (Only necessary for CircleCI Server users)
                type: string
            circleci_ip_ranges:
                default: false
                description: Enables jobs to go through a set of well-defined IP address ranges.
                type: boolean
            config-path:
                default: .circleci/continue_config.yml
                description: |
                    The location of the config to continue the pipeline with, please note that this parameter will be ignored if the user passes the config file per mapping in the mapping parameter
                type: string
            debug:
                default: false
                description: |
                    This mode is use for testing purposes. Defaults to False When this is set to True, the parameters and the config will be generated, but the continuation won't be called.
                type: boolean
            exclude:
                default: ""
                description: |
                    List of paths to exclude from the mapping. One path per line.
                type: string
            executor:
                default: default
                description: Executor where to run this job
                type: executor
            mapping:
                default: ""
                description: |
                    Mapping of path regular expressions (grep -P) to pipeline parameters and values. If the value is a file, then it will be loaded from the disk. One mapping per line, whitespace-delimited.
                type: string
            output-path:
                default: /tmp/pipeline-parameters.json
                description: |
                    Path to save the generated parameters to.
                type: string
            parameter_for_files_changed:
                default: ""
                description: |
                    If you want to get the list of files changed in the dynamic config, this value is the name of the parameter containing those files in there. Defaults to empty string. Set to empty string if you don't want the parameter.
                type: string
            resource_class:
                default: small
                description: Resource class to use
                type: string
            same-base-run:
                default: true
                description: |
                    This is value is used to know what to do when the base revision is the same as the current commit. When set to true, it will compare with the previous commit. When set to false, it will act as if not changes were done.
                type: boolean
            workspace_path:
                default: ""
                description: Path to attach the workspace to
                type: string
        resource_class: << parameters.resource_class >>
        steps:
            - when:
                condition:
                    equal:
                        - true
                        - << parameters.checkout >>
                steps:
                    - checkout:
                        method: <<parameters.checkout_method>>
            - when:
                condition:
                    not:
                        equal:
                            - ""
                            - << parameters.workspace_path >>
                steps:
                    - attach_workspace:
                        at: << parameters.workspace_path >>
            - set-parameters:
                base-revision: << parameters.base-revision >>
                config-path: << parameters.config-path >>
                exclude: <<parameters.exclude>>
                mapping: << parameters.mapping >>
                output-path: << parameters.output-path >>
                same-base-run: <<parameters.same-base-run>>
            - generate-config:
                config-list-path: /tmp/filtered-config-list
                generated-config-path: /tmp/generated-config.yml
            - when:
                condition:
                    equal:
                        - false
                        - <<parameters.debug>>
                steps:
                    - continuation/continue:
                        circleci_domain: << parameters.circleci_domain >>
                        configuration_path: /tmp/generated-config.yml
                        files_changed: /tmp/files-changed-list
                        parameter_for_files_changed: <<parameters.parameter_for_files_changed>>
                        parameters: << parameters.output-path >>
examples:
    path_filtering:
        description: |
            Continue a pipeline from the setup phase with supplied configuration and pipeline parameters generated from the files changed.
        usage:
            version: "2.1"
            setup: true
            orbs:
                path-filtering: circleci/path-filtering@0.1.7
            workflows:
                generate-config:
                    jobs:
                        - path-filtering/filter:
                            base-revision: main
                            config-path: .circleci/continue_config.yml
                            mapping: |
                                src/.* build-code true
                                doc/.* build-docs true
                                src/tests/.* string-parameter "value"
                        - path-filtering/filter:
                            base-revision: main
                            config-path: .circleci/continue_config.yml
                            mapping: |
                                src/.* build-code true .circleci/build-code-config.yml
                                doc/.* build-docs true .circleci/build-docs-config.yml
                        - path-filtering/filter:
                            base-revision: main
                            config-path: .circleci/continue-config.yml
                            mapping: .circleci/mapping.conf
                        - path-filtering/filter:
                            base-revision: main
                            config-path: .circleci/continue_config.yml
                            exclude: |
                                src/config.yml
                                build/target/.*
                            mapping: |
                                src/.*
                                doc/.* .circleci/build-docs-config.yml
                                tests/.* foo bar
                                build/.* foo bar .circleci/build.yml