Merge pull request #3372 from gilles-peskine-arm/check-files-changelog-2.16
Backport 2.16: Check changelog entries on CI
diff --git a/ChangeLog.d/bugfix.txt b/ChangeLog.d/bugfix.txt
index 5bf0aeb..d7ace30 100644
--- a/ChangeLog.d/bugfix.txt
+++ b/ChangeLog.d/bugfix.txt
@@ -1,4 +1,4 @@
Bugfix
* Fix the Visual Studio Release x64 build configuration for mbedtls itself.
- Completes a previous fix in Mbed TLS 2.16 that only fixed the build for
- the example programs. Reported in #1430 and fix contributed by irwir.
\ No newline at end of file
+ Completes a previous fix in Mbed TLS 2.16.3 that only fixed the build for
+ the example programs. Reported in #1430 and fix contributed by irwir.
diff --git a/programs/pkey/rsa_priv.txt b/programs/pkey/rsa_priv.txt
index 22c37fe..254fcf8 100644
--- a/programs/pkey/rsa_priv.txt
+++ b/programs/pkey/rsa_priv.txt
@@ -1,8 +1,8 @@
-N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211
-E = 010001
-D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401
-P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1
-Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261
-DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1
-DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081
-QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1
+N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211
+E = 010001
+D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401
+P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1
+Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261
+DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1
+DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081
+QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1
diff --git a/programs/pkey/rsa_pub.txt b/programs/pkey/rsa_pub.txt
index 2c6d313..1e7ae0c 100644
--- a/programs/pkey/rsa_pub.txt
+++ b/programs/pkey/rsa_pub.txt
@@ -1,2 +1,2 @@
-N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211
-E = 010001
+N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211
+E = 010001
diff --git a/tests/data_files/bitstring-in-dn.pem b/tests/data_files/bitstring-in-dn.pem
index 1a98aa3..c50bd66 100644
--- a/tests/data_files/bitstring-in-dn.pem
+++ b/tests/data_files/bitstring-in-dn.pem
@@ -1,51 +1,51 @@
------BEGIN CERTIFICATE-----
-MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0
-IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG
-9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp
-dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC
-WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD
-QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs
-ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk
-V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT
-SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb
-EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe
-J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt
-tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd
-iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j
-cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH
-AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA
-A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/
-A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G
-tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML
-pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE
-ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR
-5RbzoLMOxq7hoOCyIaQeM/wgxeGE
------END CERTIFICATE-----
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri
-gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2
-XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P
-NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA
-u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j
-Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v
-OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8
-2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I
-DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE
-FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq
-+Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz
-19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR
-iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL
-SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO
-/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp
-HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr
-QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr
-JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP
-GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e
-+KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU
-DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe
-FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx
-FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/
-70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an
-N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg==
------END RSA PRIVATE KEY-----
\ No newline at end of file
+-----BEGIN CERTIFICATE-----
+MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0
+IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG
+9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp
+dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC
+WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD
+QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs
+ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk
+V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT
+SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb
+EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe
+J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt
+tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd
+iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j
+cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH
+AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA
+A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/
+A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G
+tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML
+pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE
+ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR
+5RbzoLMOxq7hoOCyIaQeM/wgxeGE
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri
+gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2
+XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P
+NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA
+u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j
+Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v
+OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8
+2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I
+DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE
+FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq
++Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz
+19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR
+iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL
+SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO
+/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp
+HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr
+QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr
+JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP
+GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e
++KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU
+DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe
+FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx
+FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/
+70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an
+N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg==
+-----END RSA PRIVATE KEY-----
diff --git a/tests/data_files/test-ca.server1.opensslconf b/tests/data_files/test-ca.server1.opensslconf
index 4a5072e..209b0ff 100644
--- a/tests/data_files/test-ca.server1.opensslconf
+++ b/tests/data_files/test-ca.server1.opensslconf
@@ -1,6 +1,6 @@
[ ca ]
default_ca = test-ca
-
+
[ test-ca ]
certificate = test-ca.crt
private_key = test-ca.key
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 62156da..9c46e2c 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -589,6 +589,18 @@
record_status tests/scripts/check-files.py
}
+component_check_changelog () {
+ msg "Check: changelog entries" # < 1s
+ rm -f ChangeLog.new
+ record_status scripts/assemble_changelog.py -o ChangeLog.new
+ if [ -e ChangeLog.new ]; then
+ # Show the diff for information. It isn't an error if the diff is
+ # non-empty.
+ diff -u ChangeLog ChangeLog.new || true
+ rm ChangeLog.new
+ fi
+}
+
component_check_names () {
msg "test/build: declared and exported names" # < 3s
record_status tests/scripts/check-names.sh -v
diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py
index cd993e2..a5436fc 100755
--- a/tests/scripts/check-files.py
+++ b/tests/scripts/check-files.py
@@ -14,6 +14,8 @@
import argparse
import logging
import codecs
+import re
+import subprocess
import sys
@@ -23,28 +25,48 @@
To implement a checker that processes a file as a whole, inherit from
this class and implement `check_file_for_issue` and define ``heading``.
- ``files_exemptions``: files whose name ends with a string in this set
+ ``suffix_exemptions``: files whose name ends with a string in this set
will not be checked.
+ ``path_exemptions``: files whose path (relative to the root of the source
+ tree) matches this regular expression will not be checked. This can be
+ ``None`` to match no path. Paths are normalized and converted to ``/``
+ separators before matching.
+
``heading``: human-readable description of the issue
"""
- files_exemptions = frozenset()
+ suffix_exemptions = frozenset()
+ path_exemptions = None
# heading must be defined in derived classes.
# pylint: disable=no-member
def __init__(self):
self.files_with_issues = {}
+ @staticmethod
+ def normalize_path(filepath):
+ """Normalize ``filepath`` with / as the directory separator."""
+ filepath = os.path.normpath(filepath)
+ # On Windows, we may have backslashes to separate directories.
+ # We need slashes to match exemption lists.
+ seps = os.path.sep
+ if os.path.altsep is not None:
+ seps += os.path.altsep
+ return '/'.join(filepath.split(seps))
+
def should_check_file(self, filepath):
"""Whether the given file name should be checked.
- Files whose name ends with a string listed in ``self.files_exemptions``
- will not be checked.
+ Files whose name ends with a string listed in ``self.suffix_exemptions``
+ or whose path matches ``self.path_exemptions`` will not be checked.
"""
- for files_exemption in self.files_exemptions:
+ for files_exemption in self.suffix_exemptions:
if filepath.endswith(files_exemption):
return False
+ if self.path_exemptions and \
+ re.match(self.path_exemptions, self.normalize_path(filepath)):
+ return False
return True
def check_file_for_issue(self, filepath):
@@ -73,6 +95,17 @@
logger.info(filename)
logger.info("")
+BINARY_FILE_PATH_RE_LIST = [
+ r'docs/.*\.pdf\Z',
+ r'programs/fuzz/corpuses/[^.]+\Z',
+ r'tests/data_files/[^.]+\Z',
+ r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z',
+ r'tests/data_files/.*\.req\.[^/]+\Z',
+ r'tests/data_files/.*malformed[^/]+\Z',
+ r'tests/data_files/format_pkcs12\.fmt\Z',
+]
+BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST))
+
class LineIssueTracker(FileIssueTracker):
"""Base class for line-by-line issue tracking.
@@ -80,6 +113,9 @@
this class and implement `line_with_issue`.
"""
+ # Exclude binary files.
+ path_exemptions = BINARY_FILE_PATH_RE
+
def issue_with_line(self, line, filepath):
"""Check the specified line for the issue that this class is for.
@@ -103,7 +139,7 @@
def is_windows_file(filepath):
_root, ext = os.path.splitext(filepath)
- return ext in ('.bat', '.dsp', '.sln', '.vcxproj')
+ return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj')
class PermissionIssueTracker(FileIssueTracker):
@@ -126,9 +162,18 @@
heading = "Missing newline at end of file:"
+ path_exemptions = BINARY_FILE_PATH_RE
+
def check_file_for_issue(self, filepath):
with open(filepath, "rb") as f:
- if not f.read().endswith(b"\n"):
+ try:
+ f.seek(-1, 2)
+ except OSError:
+ # This script only works on regular files. If we can't seek
+ # 1 before the end, it means that this position is before
+ # the beginning of the file, i.e. that the file is empty.
+ return
+ if f.read(1) != b"\n":
self.files_with_issues[filepath] = None
@@ -138,7 +183,8 @@
heading = "UTF-8 BOM present:"
- files_exemptions = frozenset([".vcxproj", ".sln"])
+ suffix_exemptions = frozenset([".vcxproj", ".sln"])
+ path_exemptions = BINARY_FILE_PATH_RE
def check_file_for_issue(self, filepath):
with open(filepath, "rb") as f:
@@ -152,6 +198,8 @@
heading = "Non-Unix line endings:"
def should_check_file(self, filepath):
+ if not super().should_check_file(filepath):
+ return False
return not is_windows_file(filepath)
def issue_with_line(self, line, _filepath):
@@ -164,6 +212,8 @@
heading = "Non-Windows line endings:"
def should_check_file(self, filepath):
+ if not super().should_check_file(filepath):
+ return False
return is_windows_file(filepath)
def issue_with_line(self, line, _filepath):
@@ -174,7 +224,7 @@
"""Track lines with trailing whitespace."""
heading = "Trailing whitespace:"
- files_exemptions = frozenset([".dsp", ".md"])
+ suffix_exemptions = frozenset([".dsp", ".md"])
def issue_with_line(self, line, _filepath):
return line.rstrip(b"\r\n") != line.rstrip()
@@ -184,7 +234,8 @@
"""Track lines with tabs."""
heading = "Tabs present:"
- files_exemptions = frozenset([
+ suffix_exemptions = frozenset([
+ ".pem", # some openssl dumps have tabs
".sln",
"/Makefile",
"/generate_visualc_files.pl",
@@ -222,31 +273,6 @@
self.check_repo_path()
self.logger = None
self.setup_logger(log_file)
- self.extensions_to_check = (
- ".bat",
- ".c",
- ".data",
- ".dsp",
- ".function",
- ".h",
- ".md",
- ".pl",
- ".py",
- ".sh",
- ".sln",
- ".vcxproj",
- "/CMakeLists.txt",
- "/ChangeLog",
- "/Makefile",
- )
- self.excluded_directories = [
- '.git',
- 'mbed-os',
- ]
- self.excluded_paths = list(map(os.path.normpath, [
- 'cov-int',
- 'examples',
- ]))
self.issues_to_check = [
PermissionIssueTracker(),
EndOfFileNewlineIssueTracker(),
@@ -273,23 +299,22 @@
console = logging.StreamHandler()
self.logger.addHandler(console)
- def prune_branch(self, root, d):
- if d in self.excluded_directories:
- return True
- if os.path.normpath(os.path.join(root, d)) in self.excluded_paths:
- return True
- return False
+ @staticmethod
+ def collect_files():
+ bytes_output = subprocess.check_output(['git', 'ls-files', '-z'])
+ bytes_filepaths = bytes_output.split(b'\0')[:-1]
+ ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths)
+ # Prepend './' to files in the top-level directory so that
+ # something like `'/Makefile' in fp` matches in the top-level
+ # directory as well as in subdirectories.
+ return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp)
+ for fp in ascii_filepaths]
def check_files(self):
- for root, dirs, files in os.walk("."):
- dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d))
- for filename in sorted(files):
- filepath = os.path.join(root, filename)
- if not filepath.endswith(self.extensions_to_check):
- continue
- for issue_to_check in self.issues_to_check:
- if issue_to_check.should_check_file(filepath):
- issue_to_check.check_file_for_issue(filepath)
+ for issue_to_check in self.issues_to_check:
+ for filepath in self.collect_files():
+ if issue_to_check.should_check_file(filepath):
+ issue_to_check.check_file_for_issue(filepath)
def output_issues(self):
integrity_return_code = 0