Merge pull request #547 from OP-TED/feature/TED4-170-feedback

Dragos0000 · web-flow · commit 01673c0b8dc9 · 2024-08-14T13:17:28.000+01:00
Feature/ted4 170 feedback
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+*.jar filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -25,6 +25,7 @@ jobs:
           python-version: 3.8
       - name: Install dependencies
         run: |
+          sudo apt-get update
           sudo apt-get install -y libssl-dev libcurl4-openssl-dev
           python -m pip install --upgrade setuptools pip wheel
           make install
diff --git a/.gitignore b/.gitignore
@@ -118,4 +118,5 @@ package.json
 .limes/*
 *.ser
 .DS_Store
-.scannerwork/*
+.scannerwork/*
+/infra/alpine/libraries/
diff --git a/Makefile b/Makefile
@@ -95,6 +95,7 @@ create-env-airflow:
 	@ cp requirements.txt ./infra/airflow/
 	@ cp -r ted_sws ./infra/airflow/
 	@ cp -r dags ./infra/airflow/
+	@ cp -r libraries ./infra/airflow/
 
 
 build-airflow: guard-ENVIRONMENT create-env-airflow build-externals
diff --git a/dags/pipelines/notice_processor_pipelines.py b/dags/pipelines/notice_processor_pipelines.py
@@ -38,7 +38,8 @@ def notice_transformation_pipeline(notice: Notice, mongodb_client: MongoClient)
             message=f"This notice {notice.ted_id} is not eligible for transformation. Notice info: "
                     f"form_number=[{notice.normalised_metadata.form_number}],"
                     f" eform_subtype=[{notice.normalised_metadata.eforms_subtype}], "
-                    f"xsd_version=[{notice.normalised_metadata.xsd_version}]. Check mapping suites!",
+                    f"xsd_version=[{notice.normalised_metadata.xsd_version}], "
+                    f"eform_sdk_version=[{notice.normalised_metadata.eform_sdk_version}]. Check mapping suites!",
             notice_id=notice.ted_id, domain_action=notice_transformation_pipeline.__name__, notice_status=notice.status,
             notice_form_number=notice.normalised_metadata.form_number,
             notice_eforms_subtype=notice.normalised_metadata.eforms_subtype)
diff --git a/filters/fortify-exclusion.properties b/filters/fortify-exclusion.properties
@@ -0,0 +1 @@
+excludePatterns=**/src/docs/**/*,**/src/tests/**/*
diff --git a/filters/odc-exclusion.properties b/filters/odc-exclusion.properties
@@ -0,0 +1 @@
+excludePatterns=**/src/docs/**/*,**/src/tests/**/*
diff --git a/infra/airflow-cluster/Dockerfile b/infra/airflow-cluster/Dockerfile
@@ -15,6 +15,8 @@ RUN apt-get update && apt-get install -y \
 # back to normal user
 USER airflow
 
+COPY libraries /home/airflow
+
 # requirements.txt shall be made availble from the **ted-sws** GitHub repository
 COPY requirements.txt /opt/airflow
 
diff --git a/infra/airflow/Dockerfile b/infra/airflow/Dockerfile
@@ -15,11 +15,13 @@ RUN apt-get update && apt-get install -y \
 # back to normal user
 USER airflow
 
+COPY libraries /home/airflow
 # requirements.txt shall be made availble from the **ted-sws** GitHub repository
 COPY requirements.txt /opt/airflow
 
 # working in the /opt/airflow
 WORKDIR /opt/airflow
+
 RUN mkdir -p ./dags ./ted_sws
 
 
diff --git a/libraries/.limes/limes.jar b/libraries/.limes/limes.jar
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88363c8d83f41d7a6dec9ea9b66cb4a72dd2f022343f58ec03dfb814722b43b6
+size 128740593
diff --git a/libraries/.rmlmapper/rmlmapper.jar b/libraries/.rmlmapper/rmlmapper.jar
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28b3e1b6b5718768e9465ddddee204d127889ddd02db5736913d959009e765d0
+size 77468637
diff --git a/libraries/.saxon/lib/jline-2.14.6.jar b/libraries/.saxon/lib/jline-2.14.6.jar
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97d1acaac82409be42e622d7a54d3ae9d08517e8aefdea3d2ba9791150c2f02d
+size 268780
diff --git a/libraries/.saxon/lib/xml-resolver-1.2.jar b/libraries/.saxon/lib/xml-resolver-1.2.jar
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47dcde8986019314ef78ae7280a94973a21d2ed95075a40a000b42da956429e1
+size 84091
diff --git a/libraries/.saxon/notices/JAMESCLARK.txt b/libraries/.saxon/notices/JAMESCLARK.txt
@@ -0,0 +1,31 @@
+
+(This notice is included in the Saxon distribution because Saxon's XPath parser
+was originally derived from an XPath parser written by James Clark and made available
+under this license. The Saxon XPath parser has since diverged very substantially, but
+there are traces of the original code still present.)
+
+Copyright (c) 1998, 1999 James Clark
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL JAMES CLARK BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of James Clark shall
+not be used in advertising or otherwise to promote the sale, use or
+other dealings in this Software without prior written authorization
+from James Clark.
diff --git a/libraries/.saxon/notices/JLINE2.txt b/libraries/.saxon/notices/JLINE2.txt
@@ -0,0 +1,39 @@
+(This notice is included in the Saxon distribution because the JLine2
+library is included, unmodified, in the Saxon distribution. It is used
+to support interactive command line editing, autocompletion, and session
+history by the Gizmo utility.)
+
+Copyright (c) 2002-2016, the original author or authors.
+All rights reserved.
+
+http://www.opensource.org/licenses/bsd-license.php
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the following
+conditions are met:
+
+Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with
+the distribution.
+
+Neither the name of JLine nor the names of its contributors
+may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/libraries/.saxon/notices/LICENSE.txt b/libraries/.saxon/notices/LICENSE.txt
diff --git a/libraries/.saxon/notices/THAI.txt b/libraries/.saxon/notices/THAI.txt
diff --git a/libraries/.saxon/notices/UNICODE.txt b/libraries/.saxon/notices/UNICODE.txt
diff --git a/libraries/.saxon/saxon-he-10.9.jar b/libraries/.saxon/saxon-he-10.9.jar
diff --git a/libraries/.saxon/saxon-he-test-10.9.jar b/libraries/.saxon/saxon-he-test-10.9.jar
diff --git a/libraries/.saxon/saxon-he-xqj-10.9.jar b/libraries/.saxon/saxon-he-xqj-10.9.jar

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+*.jar filter=lfs diff=lfs merge=lfs -text`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+excludePatterns=/src/docs//,/src/tests//`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:88363c8d83f41d7a6dec9ea9b66cb4a72dd2f022343f58ec03dfb814722b43b6`
	`3`	`+size 128740593`