blob: 52c4f8f0e8bed47e6cdc19a952a34b991a0537e6 [file] [log] [blame]
Laurence Lundbladef156fb82018-10-01 09:47:03 -07001/*==============================================================================
2 run_tests.c -- test aggregator and results reporting
Laurence Lundblade3aee3a32018-12-17 16:17:45 -08003
Laurence Lundbladea3fd49f2019-01-21 10:16:22 -08004 Copyright (c) 2018-2019, Laurence Lundblade. All rights reserved.
Laurence Lundblade035bd782019-01-21 17:01:31 -08005
Laurence Lundbladea3fd49f2019-01-21 10:16:22 -08006 SPDX-License-Identifier: BSD-3-Clause
Laurence Lundblade035bd782019-01-21 17:01:31 -08007
Laurence Lundbladea3fd49f2019-01-21 10:16:22 -08008 See BSD-3-Clause license in README.md
Laurence Lundbladef156fb82018-10-01 09:47:03 -07009
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -080010 Created on 9/30/18
Laurence Lundblade88b6ece2018-12-04 12:27:19 +090011 ==============================================================================*/
Laurence Lundbladef156fb82018-10-01 09:47:03 -070012
Laurence Lundbladef156fb82018-10-01 09:47:03 -070013#include "run_tests.h"
14#include "UsefulBuf.h"
15#include <stdbool.h>
16
Laurence Lundblade2d85ce42018-10-12 14:12:47 +080017#include "float_tests.h"
Laurence Lundblade9e3651c2018-10-10 11:49:55 +080018#include "qcbor_decode_tests.h"
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053019#include "qcbor_encode_tests.h"
20#include "UsefulBuf_Tests.h"
Laurence Lundbladef156fb82018-10-01 09:47:03 -070021
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -080022
Laurence Lundbladef156fb82018-10-01 09:47:03 -070023/*
Laurence Lundbladea65b4822019-09-08 12:17:03 -070024 Test configuration
25 */
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -080026
Laurence Lundbladef156fb82018-10-01 09:47:03 -070027typedef int (test_fun_t)(void);
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053028typedef const char * (test_fun2_t)(void);
29
Laurence Lundbladef156fb82018-10-01 09:47:03 -070030
Laurence Lundbladea2e29072018-12-30 09:20:06 -080031#define TEST_ENTRY(test_name) {#test_name, test_name, true}
32#define TEST_ENTRY_DISABLED(test_name) {#test_name, test_name, false}
33
Laurence Lundbladef156fb82018-10-01 09:47:03 -070034typedef struct {
Laurence Lundbladea2e29072018-12-30 09:20:06 -080035 const char *szTestName;
Laurence Lundbladef156fb82018-10-01 09:47:03 -070036 test_fun_t *test_fun;
Laurence Lundbladea2e29072018-12-30 09:20:06 -080037 bool bEnabled;
Laurence Lundbladef156fb82018-10-01 09:47:03 -070038} test_entry;
39
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053040typedef struct {
41 const char *szTestName;
42 test_fun2_t *test_fun;
Laurence Lundbladea2e29072018-12-30 09:20:06 -080043 bool bEnabled;
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053044} test_entry2;
45
Laurence Lundbladea65b4822019-09-08 12:17:03 -070046
47static test_entry2 s_tests2[] = {
Laurence Lundblade7566b9f2018-10-12 09:13:32 +080048 TEST_ENTRY(UBUTest_CopyUtil),
49 TEST_ENTRY(UOBTest_NonAdversarial),
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053050 TEST_ENTRY(TestBasicSanity),
Laurence Lundblade7566b9f2018-10-12 09:13:32 +080051 TEST_ENTRY(UOBTest_BoundaryConditionsTest),
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053052 TEST_ENTRY(UBMacroConversionsTest),
53 TEST_ENTRY(UBUtilTests),
Laurence Lundblade7566b9f2018-10-12 09:13:32 +080054 TEST_ENTRY(UIBTest_IntegerFormat)
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053055};
56
57
Laurence Lundbladea65b4822019-09-08 12:17:03 -070058static test_entry s_tests[] = {
Laurence Lundblade9916b1b2019-09-07 22:33:25 -070059 TEST_ENTRY(EmptyMapsAndArraysTest),
Laurence Lundbladebb1062e2019-08-12 23:28:54 -070060 TEST_ENTRY(NotWellFormedTests),
Laurence Lundbladeccfb8cd2018-12-07 21:11:30 +090061 TEST_ENTRY(ParseMapAsArrayTest),
Laurence Lundbladea44d5062018-10-17 18:45:12 +053062 TEST_ENTRY(AllocAllStringsTest),
63 TEST_ENTRY(IndefiniteLengthNestTest),
64 TEST_ENTRY(NestedMapTestIndefLen),
65 TEST_ENTRY(ParseSimpleTest),
Laurence Lundblade3a6042e2019-06-28 19:58:04 -070066 TEST_ENTRY(DecodeFailureTests),
Laurence Lundblade0fb6c6d2018-10-12 22:02:05 +080067 TEST_ENTRY(EncodeRawTest),
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +053068 TEST_ENTRY(RTICResultsTest),
69 TEST_ENTRY(MapEncodeTest),
70 TEST_ENTRY(ArrayNestingTest1),
71 TEST_ENTRY(ArrayNestingTest2),
72 TEST_ENTRY(ArrayNestingTest3),
73 TEST_ENTRY(EncodeDateTest),
74 TEST_ENTRY(SimpleValuesTest1),
75 TEST_ENTRY(IntegerValuesTest1),
76 TEST_ENTRY(AllAddMethodsTest),
Laurence Lundblade9e3651c2018-10-10 11:49:55 +080077 TEST_ENTRY(ParseTooDeepArrayTest),
78 TEST_ENTRY(ComprehensiveInputTest),
79 TEST_ENTRY(ParseMapTest),
Laurence Lundbladea44d5062018-10-17 18:45:12 +053080 TEST_ENTRY(IndefiniteLengthArrayMapTest),
Laurence Lundblade369b90a2018-10-22 02:04:37 +053081 TEST_ENTRY(BasicEncodeTest),
Laurence Lundblade9e3651c2018-10-10 11:49:55 +080082 TEST_ENTRY(NestedMapTest),
83 TEST_ENTRY(BignumParseTest),
84 TEST_ENTRY(OptTagParseTest),
85 TEST_ENTRY(DateParseTest),
Laurence Lundblade9e3651c2018-10-10 11:49:55 +080086 TEST_ENTRY(ShortBufferParseTest2),
87 TEST_ENTRY(ShortBufferParseTest),
88 TEST_ENTRY(ParseDeepArrayTest),
89 TEST_ENTRY(SimpleArrayTest),
90 TEST_ENTRY(IntegerValuesParseTest),
Laurence Lundbladea44d5062018-10-17 18:45:12 +053091 TEST_ENTRY(MemPoolTest),
92 TEST_ENTRY(IndefiniteLengthStringTest),
Laurence Lundbladebb474be2018-10-22 11:53:21 +053093 TEST_ENTRY(HalfPrecisionDecodeBasicTests),
Laurence Lundbladebb474be2018-10-22 11:53:21 +053094 TEST_ENTRY(DoubleAsSmallestTest),
95 TEST_ENTRY(HalfPrecisionAgainstRFCCodeTest),
Laurence Lundblade369b90a2018-10-22 02:04:37 +053096 TEST_ENTRY(BstrWrapTest),
97 TEST_ENTRY(BstrWrapErrorTest),
98 TEST_ENTRY(BstrWrapNestTest),
99 TEST_ENTRY(CoseSign1TBSTest),
Laurence Lundbladeea567ac2018-12-09 14:03:21 -0800100 TEST_ENTRY(StringDecoderModeFailTest),
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800101 TEST_ENTRY_DISABLED(BigComprehensiveInputTest),
Laurence Lundblade1ef8b2d2018-12-14 23:13:34 -0800102 TEST_ENTRY(EncodeErrorTests),
Laurence Lundblade1d7eb632019-02-17 17:23:38 -0800103 TEST_ENTRY(SetUpAllocatorTest),
Jan Jongboom47d86c52019-07-25 08:54:16 +0200104 TEST_ENTRY(SimpleValuesIndefiniteLengthTest1),
Jan Jongboom5d827882019-08-07 12:51:15 +0200105 TEST_ENTRY(EncodeLengthThirtyoneTest),
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700106};
107
108
Laurence Lundbladea65b4822019-09-08 12:17:03 -0700109
110
111/*
112 Convert a number up to 999999999 to a string. This is so sprintf doesn't
113 have to be linked in so as to minimized dependencies even in test code.
Laurence Lundblade6216b1b2019-09-27 11:24:39 +0900114
115 StringMem should be 12 bytes long, 9 for digits, 1 for minus and
116 1 for \0 termination.
Laurence Lundbladea65b4822019-09-08 12:17:03 -0700117 */
118static const char *NumToString(int32_t nNum, UsefulBuf StringMem)
119{
120 const int32_t nMax = 1000000000;
121
122 UsefulOutBuf OutBuf;
123 UsefulOutBuf_Init(&OutBuf, StringMem);
124
125 if(nNum < 0) {
126 UsefulOutBuf_AppendByte(&OutBuf, '-');
127 nNum = -nNum;
128 }
129 if(nNum > nMax-1) {
130 return "XXX";
131 }
132
133 bool bDidSomeOutput = false;
134 for(int n = nMax; n > 0; n/=10) {
135 int x = nNum/n;
136 if(x || bDidSomeOutput){
137 bDidSomeOutput = true;
138 UsefulOutBuf_AppendByte(&OutBuf, '0' + x);
139 nNum -= x * n;
140 }
141 }
142 if(!bDidSomeOutput){
143 UsefulOutBuf_AppendByte(&OutBuf, '0');
144 }
145 UsefulOutBuf_AppendByte(&OutBuf, '\0');
146
147 return UsefulOutBuf_GetError(&OutBuf) ? "" : StringMem.ptr;
148}
149
150
151/*
152 Public function. See run_test.h.
153 */
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800154int RunTests(const char *szTestNames[], OutputStringCB pfOutput, void *poutCtx, int *pNumTestsRun)
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700155{
156 int nTestsFailed = 0;
157 int nTestsRun = 0;
Laurence Lundblade6216b1b2019-09-27 11:24:39 +0900158 UsefulBuf_MAKE_STACK_UB(StringStorage, 12);
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530159
160 test_entry2 *t2;
161 const test_entry2 *s_tests2_end = s_tests2 + sizeof(s_tests2)/sizeof(test_entry2);
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800162
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530163 for(t2 = s_tests2; t2 < s_tests2_end; t2++) {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800164 if(szTestNames[0]) {
165 // Some tests have been named
166 const char **szRequestedNames;
167 for(szRequestedNames = szTestNames; *szRequestedNames; szRequestedNames++) {
168 if(!strcmp(t2->szTestName, *szRequestedNames)) {
169 break; // Name matched
170 }
171 }
172 if(*szRequestedNames == NULL) {
173 // Didn't match this test
174 continue;
175 }
176 } else {
177 // no tests named, but don't run "disabled" tests
178 if(!t2->bEnabled) {
179 // Don't run disabled tests when all tests are being run
180 // as indicated by no specific test names being given
181 continue;
182 }
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800183 }
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800184
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800185 const char * szTestResult = (t2->test_fun)();
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530186 nTestsRun++;
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800187 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800188 (*pfOutput)(t2->szTestName, poutCtx, 0);
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530189 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800190
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800191 if(szTestResult) {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800192 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800193 (*pfOutput)(" FAILED (returned ", poutCtx, 0);
194 (*pfOutput)(szTestResult, poutCtx, 0);
Laurence Lundbladedf1c1cf2019-01-17 11:55:05 -0800195 (*pfOutput)(")", poutCtx, 1);
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530196 }
197 nTestsFailed++;
198 } else {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800199 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800200 (*pfOutput)( " PASSED", poutCtx, 1);
Laurence Lundbladedc6e28e2018-10-11 19:19:27 +0530201 }
202 }
203 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800204
205
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700206 test_entry *t;
207 const test_entry *s_tests_end = s_tests + sizeof(s_tests)/sizeof(test_entry);
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800208
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700209 for(t = s_tests; t < s_tests_end; t++) {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800210 if(szTestNames[0]) {
211 // Some tests have been named
212 const char **szRequestedNames;
213 for(szRequestedNames = szTestNames; *szRequestedNames; szRequestedNames++) {
214 if(!strcmp(t->szTestName, *szRequestedNames)) {
215 break; // Name matched
216 }
217 }
218 if(*szRequestedNames == NULL) {
219 // Didn't match this test
220 continue;
221 }
222 } else {
223 // no tests named, but don't run "disabled" tests
224 if(!t->bEnabled) {
225 // Don't run disabled tests when all tests are being run
226 // as indicated by no specific test names being given
227 continue;
228 }
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800229 }
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800230
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800231 int nTestResult = (t->test_fun)();
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700232 nTestsRun++;
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800233 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800234 (*pfOutput)(t->szTestName, poutCtx, 0);
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700235 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800236
Laurence Lundbladea2e29072018-12-30 09:20:06 -0800237 if(nTestResult) {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800238 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800239 (*pfOutput)(" FAILED (returned ", poutCtx, 0);
240 (*pfOutput)(NumToString(nTestResult, StringStorage), poutCtx, 0);
241 (*pfOutput)(")", poutCtx, 1);
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700242 }
243 nTestsFailed++;
244 } else {
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800245 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800246 (*pfOutput)( " PASSED", poutCtx, 1);
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700247 }
248 }
249 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800250
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700251 if(pNumTestsRun) {
252 *pNumTestsRun = nTestsRun;
253 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800254
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800255 if(pfOutput) {
Laurence Lundbladed396f622019-01-12 17:12:29 -0800256 (*pfOutput)( "SUMMARY: ", poutCtx, 0);
257 (*pfOutput)( NumToString(nTestsRun, StringStorage), poutCtx, 0);
258 (*pfOutput)( " tests run; ", poutCtx, 0);
259 (*pfOutput)( NumToString(nTestsFailed, StringStorage), poutCtx, 0);
260 (*pfOutput)( " tests failed", poutCtx, 1);
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700261 }
Laurence Lundblade3aee3a32018-12-17 16:17:45 -0800262
Laurence Lundbladef156fb82018-10-01 09:47:03 -0700263 return nTestsFailed;
264}
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800265
266
Laurence Lundbladea65b4822019-09-08 12:17:03 -0700267#include "qcbor.h" // For size printing
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800268
Laurence Lundbladea65b4822019-09-08 12:17:03 -0700269/*
270 Public function. See run_test.h.
271 */
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800272static void PrintSize(const char *szWhat, uint32_t uSize, OutputStringCB pfOutput, void *pOutCtx)
273{
Laurence Lundbladed396f622019-01-12 17:12:29 -0800274 UsefulBuf_MAKE_STACK_UB(buffer, 20);
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800275
Laurence Lundbladed396f622019-01-12 17:12:29 -0800276 (*pfOutput)(szWhat, pOutCtx, 0);
277 (*pfOutput)(" ", pOutCtx, 0);
278 (*pfOutput)(NumToString(uSize, buffer), pOutCtx, 0);
279 (*pfOutput)("", pOutCtx, 1);
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800280}
281
Laurence Lundbladea65b4822019-09-08 12:17:03 -0700282
283/*
284 Public function. See run_test.h.
285 */
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800286void PrintSizes(OutputStringCB pfOutput, void *pOutCtx)
287{
Laurence Lundbladed396f622019-01-12 17:12:29 -0800288 // Type and size of return from sizeof() varies. These will never be large so cast is safe
289 PrintSize("sizeof(QCBORTrackNesting)", (uint32_t)sizeof(QCBORTrackNesting), pfOutput, pOutCtx);
290 PrintSize("sizeof(QCBOREncodeContext)", (uint32_t)sizeof(QCBOREncodeContext), pfOutput, pOutCtx);
291 PrintSize("sizeof(QCBORDecodeNesting)", (uint32_t)sizeof(QCBORDecodeNesting), pfOutput, pOutCtx);
292 PrintSize("sizeof(QCBORDecodeContext)", (uint32_t)sizeof(QCBORDecodeContext), pfOutput, pOutCtx);
293 PrintSize("sizeof(QCBORItem)", (uint32_t)sizeof(QCBORItem), pfOutput, pOutCtx);
Laurence Lundbladed396f622019-01-12 17:12:29 -0800294 PrintSize("sizeof(QCBORTagListIn)", (uint32_t)sizeof(QCBORTagListIn), pfOutput, pOutCtx);
295 PrintSize("sizeof(QCBORTagListOut)", (uint32_t)sizeof(QCBORTagListOut), pfOutput, pOutCtx);
296 (*pfOutput)("", pOutCtx, 1);
Laurence Lundblade1f8b5b02019-01-01 22:27:38 -0800297}