Skip to content

Commit 7bea95b

Browse files
hhasijahhasija
authored
fix: Jenkins optimisation update (#1024)
* fix: adding blogpost link for BigQuery to JDBC * fix: including pom.xml and jenkins file for running test cases * fix: making changes in UI Jenkins file as well * fix: changes made for python jenkins file as well * fix: including requirements.txt file of python as well
1 parent 3c5dc82 commit 7bea95b

File tree

3 files changed

+88
-88
lines changed

3 files changed

+88
-88
lines changed

‎java/.ci/Jenkinsfile‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,9 @@ EOF
101101
stage('JDBC TO BQ'){
102102
when {
103103
expression {
104-
// Check if there are any changes in Java files in the last commit
104+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
105105
return sh(
106-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
106+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
107107
returnStatus: true
108108
) == 0
109109
}
@@ -130,9 +130,9 @@ EOF
130130
stage('Parallel Execution 1'){
131131
when {
132132
expression {
133-
// Check if there are any changes in Java files in the last commit
133+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
134134
return sh(
135-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
135+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
136136
returnStatus: true
137137
) == 0
138138
}
@@ -262,9 +262,9 @@ EOF
262262
stage('Parallel Execution 2'){
263263
when {
264264
expression {
265-
// Check if there are any changes in Java files in the last commit
265+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
266266
return sh(
267-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
267+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
268268
returnStatus: true
269269
) == 0
270270
}
@@ -391,9 +391,9 @@ EOF
391391
stage('Parallel Execution 3'){
392392
when {
393393
expression {
394-
// Check if there are any changes in Java files in the last commit
394+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
395395
return sh(
396-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
396+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
397397
returnStatus: true
398398
) == 0
399399
}
@@ -494,9 +494,9 @@ EOF
494494
stage('Parallel Execution 4'){
495495
when {
496496
expression {
497-
// Check if there are any changes in Java files in the last commit
497+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
498498
return sh(
499-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
499+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
500500
returnStatus: true
501501
) == 0
502502
}
@@ -603,9 +603,9 @@ EOF
603603
stage('Parallel Execution 5'){
604604
when {
605605
expression {
606-
// Check if there are any changes in Java files in the last commit
606+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
607607
return sh(
608-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
608+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
609609
returnStatus: true
610610
) == 0
611611
}
@@ -699,9 +699,9 @@ EOF
699699
stage('Parallel Execution 6'){
700700
when {
701701
expression {
702-
// Check if there are any changes in Java files in the last commit
702+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
703703
return sh(
704-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
704+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
705705
returnStatus: true
706706
) == 0
707707
}
@@ -802,9 +802,9 @@ EOF
802802
stage('Parallel Execution 7'){
803803
when {
804804
expression {
805-
// Check if there are any changes in Java files in the last commit
805+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
806806
return sh(
807-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
807+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
808808
returnStatus: true
809809
) == 0
810810
}
@@ -884,9 +884,9 @@ EOF
884884
stage('Parallel Execution 8'){
885885
when {
886886
expression {
887-
// Check if there are any changes in Java files in the last commit
887+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
888888
return sh(
889-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
889+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
890890
returnStatus: true
891891
) == 0
892892
}
@@ -989,9 +989,9 @@ EOF
989989
stage('Parallel Execution 9'){
990990
when {
991991
expression {
992-
// Check if there are any changes in Java files in the last commit
992+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
993993
return sh(
994-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
994+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
995995
returnStatus: true
996996
) == 0
997997
}
@@ -1105,9 +1105,9 @@ EOF
11051105
stage('Parallel Execution 10'){
11061106
when {
11071107
expression {
1108-
// Check if there are any changes in Java files in the last commit
1108+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
11091109
return sh(
1110-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
1110+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
11111111
returnStatus: true
11121112
) == 0
11131113
}

‎java/.ci/UiJenkinsfile‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,9 @@ pipeline {
7878
stage('Parallel Execution 1'){
7979
when {
8080
expression {
81-
// Check if there are any changes in Java files in the last commit
81+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
8282
return sh(
83-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
83+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
8484
returnStatus: true
8585
) == 0
8686
}
@@ -184,9 +184,9 @@ pipeline {
184184
stage('Parallel Execution 2'){
185185
when {
186186
expression {
187-
// Check if there are any changes in Java files in the last commit
187+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
188188
return sh(
189-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
189+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
190190
returnStatus: true
191191
) == 0
192192
}
@@ -286,9 +286,9 @@ pipeline {
286286
stage('Parallel Execution 3'){
287287
when {
288288
expression {
289-
// Check if there are any changes in Java files in the last commit
289+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
290290
return sh(
291-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
291+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
292292
returnStatus: true
293293
) == 0
294294
}
@@ -366,9 +366,9 @@ pipeline {
366366
stage('Parallel Execution 4'){
367367
when {
368368
expression {
369-
// Check if there are any changes in Java files in the last commit
369+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
370370
return sh(
371-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
371+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
372372
returnStatus: true
373373
) == 0
374374
}
@@ -424,9 +424,9 @@ pipeline {
424424
stage('Parallel Execution 5'){
425425
when {
426426
expression {
427-
// Check if there are any changes in Java files in the last commit
427+
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
428428
return sh(
429-
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
429+
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
430430
returnStatus: true
431431
) == 0
432432
}

‎python/.ci/Jenkinsfile‎

Lines changed: 56 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,14 @@ pipeline {
6969
}
7070
stage('Parallel Execution 1'){
7171
when {
72-
expression {
73-
// Check if there are any changes in Python files in the last commit
74-
return sh(
75-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
76-
returnStatus: true
77-
) == 0
72+
expression {
73+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
74+
return sh(
75+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
76+
returnStatus: true
77+
) == 0
78+
}
7879
}
79-
}
8080
parallel{
8181
stage('Hive TO GCS') {
8282
steps{
@@ -165,14 +165,14 @@ pipeline {
165165
}
166166
stage('Parallel Execution 2'){
167167
when {
168-
expression {
169-
// Check if there are any changes in Python files in the last commit
170-
return sh(
171-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
172-
returnStatus: true
173-
) == 0
168+
expression {
169+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
170+
return sh(
171+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
172+
returnStatus: true
173+
) == 0
174+
}
174175
}
175-
}
176176
parallel{
177177
stage('JDBC To JDBC') {
178178
steps{
@@ -257,14 +257,14 @@ pipeline {
257257
}
258258
stage('Parallel Execution 3'){
259259
when {
260-
expression {
261-
// Check if there are any changes in Python files in the last commit
262-
return sh(
263-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
264-
returnStatus: true
265-
) == 0
260+
expression {
261+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
262+
return sh(
263+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
264+
returnStatus: true
265+
) == 0
266+
}
266267
}
267-
}
268268
parallel{
269269
stage('GCS TO BIGQUERY') {
270270
steps{
@@ -353,14 +353,14 @@ pipeline {
353353
}
354354
stage('Parallel Execution 4'){
355355
when {
356-
expression {
357-
// Check if there are any changes in Python files in the last commit
358-
return sh(
359-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
360-
returnStatus: true
361-
) == 0
356+
expression {
357+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
358+
return sh(
359+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
360+
returnStatus: true
361+
) == 0
362+
}
362363
}
363-
}
364364
parallel{
365365
stage('MONGO TO GCS') {
366366
steps{
@@ -514,14 +514,14 @@ pipeline {
514514
}
515515
stage('Parallel Execution 5'){
516516
when {
517-
expression {
518-
// Check if there are any changes in Python files in the last commit
519-
return sh(
520-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
521-
returnStatus: true
522-
) == 0
517+
expression {
518+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
519+
return sh(
520+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
521+
returnStatus: true
522+
) == 0
523+
}
523524
}
524-
}
525525
parallel{
526526
stage('JDBC TO GCS') {
527527
steps{
@@ -614,14 +614,14 @@ pipeline {
614614
}
615615
stage('Parallel Execution 6'){
616616
when {
617-
expression {
618-
// Check if there are any changes in Python files in the last commit
619-
return sh(
620-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
621-
returnStatus: true
622-
) == 0
617+
expression {
618+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
619+
return sh(
620+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
621+
returnStatus: true
622+
) == 0
623+
}
623624
}
624-
}
625625
parallel{
626626
stage('S3 TO BigQuery (avro)') {
627627
steps{
@@ -659,14 +659,14 @@ pipeline {
659659
}
660660
stage('Parallel Execution 7'){
661661
when {
662-
expression {
663-
// Check if there are any changes in Python files in the last commit
664-
return sh(
665-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
666-
returnStatus: true
667-
) == 0
662+
expression {
663+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
664+
return sh(
665+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
666+
returnStatus: true
667+
) == 0
668+
}
668669
}
669-
}
670670
parallel{
671671
// stage('Hbase TO GCS (Manual)') {
672672
// when {
@@ -790,14 +790,14 @@ pipeline {
790790
}
791791
stage('Parallel Execution 8'){
792792
when {
793-
expression {
794-
// Check if there are any changes in Python files in the last commit
795-
return sh(
796-
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
797-
returnStatus: true
798-
) == 0
793+
expression {
794+
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
795+
return sh(
796+
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
797+
returnStatus: true
798+
) == 0
799+
}
799800
}
800-
}
801801
parallel{
802802
stage('Elasticsearch to GCS'){
803803
steps{

0 commit comments

Comments
 (0)