Precision-preserving number decode and conversion (#211)
This is part of dCBOR support. It decodes a number (int or float) into the best C representation. It's good for more than just dCBOR.
* Remove more stuff related to QCBOREncode_AddBytesLenOnly
* Commit beginnings so dev can be merged in
* Precision-preserving number decode
* Fix ifdefs for precise number decoding
* blank lines
* blank lines
* 65-bit negs without float HW
---------
Co-authored-by: Laurence Lundblade <lgl@securitytheory.com>
diff --git a/README.md b/README.md
index 635211b..e6655cb 100644
--- a/README.md
+++ b/README.md
@@ -335,23 +335,23 @@
defining this is to remove dependency on floating point hardware and
libraries.
+
#### #define QCBOR_DISABLE_PREFERRED_FLOAT
-This eliminates support for half-precision
-and CBOR preferred serialization by disabling
-QCBOR's shift and mask based implementation of
-half-precision floating-point.
+This eliminates support of:
+- encode/decode of half-precision
+- shortest-form encoding of floats
+- QCBORDecode_GetNumberConvertPrecisely()
-With this defined, single and double-precision floating-point
-numbers can still be encoded and decoded. Conversion
-of floating-point to and from integers, big numbers and
-such is also supported. Floating-point dates are still
-supported.
+This saves about 1KB of object code, though much of this can be saved
+by not calling any functions to encode doubles or floats or
+QCBORDecode_GetNumberConvertPrecisely
-The primary reason to define this is to save object code.
-Roughly 900 bytes are saved, though about half of this
-can be saved just by not calling any functions that
-encode floating-point numbers.
+With this defined, single and double-precision floating-point numbers
+can still be encoded and decoded. Some conversion of floating-point to
+and from integers, big numbers and such is also supported. Floating-point
+dates are still supported.
+
#### #define USEFULBUF_DISABLE_ALL_FLOAT
diff --git a/inc/qcbor/qcbor_spiffy_decode.h b/inc/qcbor/qcbor_spiffy_decode.h
index 84fb516..1370ada 100644
--- a/inc/qcbor/qcbor_spiffy_decode.h
+++ b/inc/qcbor/qcbor_spiffy_decode.h
@@ -300,6 +300,54 @@
int64_t *pnValue);
+#if !defined(USEFULBUF_DISABLE_ALL_FLOAT) && !defined(QCBOR_DISABLE_PREFERRED_FLOAT)
+/**
+ * @brief Decode next as a number with precision-preserving conversions.
+ *
+ * @param[in] pCtx The decode context.
+ * @param[out] pNumber The returned 64-bit signed integer.
+ *
+ * This will get the next item as a number and return it as a C
+ * data type such that no precision is lost.
+ *
+ * The CBOR input can be integers (major type 0 or 1) or floats (major type 7).
+ * If not these \ref QCBOR_ERR_UNEXPECTED_TYPE will be set.
+ *
+ * The conversion is as follows.
+ *
+ * Whole numbers between \c INT64_MIN and \c INT64_MAX will
+ * be returned as \ref QCBOR_TYPE_INT64. This includes
+ * conversion of float-point values that are whole numbers.
+ *
+ * Whole numbers between \c INT64_MAX and \c UINT64_MAX will
+ * be returned as \ref QCBOR_TYPE_UINT64, again including
+ * conversion of floating-point values that are whole numbers.
+ *
+ * The whole numbers called "65-bit negative" in CBOR (-2^63 - 1 to -2^64) are a
+ * special case. Some of them can be converted to a double without
+ * loss of precision and some can't (uint64_t has 64 bits of precision; a double
+ * has only 52). If they can't be converted to a double, they are returned
+ * as \ref QCBOR_TYPE_65BIT_NEG_INT.
+ * In many cases, it will be reasonable to error out if the
+ * number type returned here is \ref QCBOR_TYPE_65BIT_NEG_INT
+ * on the assumption that many protocols will never uses these.
+ * See also QCBOREncode_AddNegativeUInt64() for more discussion.
+ *
+ * All others are returned as a double.
+ *
+ * This is useful for dCBOR which essentially combines floats
+ * and integers into one number space.
+ *
+ * Please see @ref Decode-Errors-Overview "Decode Errors Overview".
+ *
+ * See also QCBORDecode_GetNumberConvertPreciselyBig().
+ */
+void
+QCBORDecode_GetNumberConvertPrecisely(QCBORDecodeContext *pCtx,
+ QCBORItem *pNumber);
+
+#endif /* ! USEFULBUF_DISABLE_ALL_FLOAT && ! QCBOR_DISABLE_PREFERRED_FLOAT */
+
/**
* @brief Decode next item into a signed 64-bit integer with conversions.
*
diff --git a/src/ieee754.c b/src/ieee754.c
index 002ca40..69bf113 100644
--- a/src/ieee754.c
+++ b/src/ieee754.c
@@ -191,6 +191,7 @@
* This returns the bits for a single-precision float, a binary64
* as specified in IEEE754.
*/
+// TODO: make the sign and exponent type int?
static double
IEEE754_AssembleDouble(uint64_t uDoubleSign,
uint64_t uDoubleSignificand,
@@ -644,27 +645,40 @@
}
+
+/* This returns 64 minus the number of zero bits on the right. It is
+ * is the amount of precision in the 64-bit significand passed in.
+ * When used for 52 and 23-bit significands, subtract 12 and 41
+ * to get their precision.
+ *
+ * The value returned is for a *normalized* number like the
+ * significand of a double. When used for precision for a non-normalized
+ * number like a uint64_t, further computation is required.
+ *
+ * If the significand is 0, then 0 is returned as the precision.*/
static int
-IEEE754_Private_CountNonZeroBits(int nMax, uint64_t uTarget)
+IEEE754_Private_CountPrecisionBits(uint64_t uSignigicand)
{
int nNonZeroBitsCount;
uint64_t uMask;
- for(nNonZeroBitsCount = nMax; nNonZeroBitsCount > 0; nNonZeroBitsCount--) {
- uMask = (0x01UL << nMax) >> nNonZeroBitsCount;
- if(uMask & uTarget) {
+ for(nNonZeroBitsCount = 64; nNonZeroBitsCount > 0; nNonZeroBitsCount--) {
+ uMask = 0x01UL << (64 - nNonZeroBitsCount);
+ if(uMask & uSignigicand) {
break;
}
}
+
return nNonZeroBitsCount;
}
+
/* Public function; see ieee754.h */
struct IEEE754_ToInt
IEEE754_DoubleToInt(const double d)
{
- int64_t nNonZeroBitsCount;
+ int64_t nPrecisionBits;
struct IEEE754_ToInt Result;
uint64_t uInteger;
@@ -700,19 +714,15 @@
/* --- Exponent out of range --- */
Result.type = IEEE754_ToInt_NO_CONVERSION;
} else {
- /* Count down from 52 to the number of bits that are not zero in
- * the significand. This counts from the least significant bit
- * until a non-zero bit is found to know if it is a whole
- * number.
- *
- * Conversion only fails when the input is too large or is not a
+ /* Conversion only fails when the input is too large or is not a
* whole number, never because of lack of precision because
* 64-bit integers always have more precision than the 52-bits
* of a double.
*/
- nNonZeroBitsCount = IEEE754_Private_CountNonZeroBits(DOUBLE_NUM_SIGNIFICAND_BITS, uDoubleSignificand);
+ nPrecisionBits = IEEE754_Private_CountPrecisionBits(uDoubleSignificand) -
+ (64-DOUBLE_NUM_SIGNIFICAND_BITS);
- if(nNonZeroBitsCount && nNonZeroBitsCount > nDoubleUnbiasedExponent) {
+ if(nPrecisionBits && nPrecisionBits > nDoubleUnbiasedExponent) {
/* --- Not a whole number --- */
Result.type = IEEE754_ToInt_NO_CONVERSION;
} else {
@@ -746,7 +756,7 @@
struct IEEE754_ToInt
IEEE754_SingleToInt(const float f)
{
- int32_t nNonZeroBitsCount;
+ int32_t nPrecisionBits;
struct IEEE754_ToInt Result;
uint64_t uInteger;
@@ -781,18 +791,15 @@
/* --- Exponent out of range --- */
Result.type = IEEE754_ToInt_NO_CONVERSION;
} else {
- /* Count down from 23 to the number of bits that are not zero in
- * the significand. This counts from the least significant bit
- * until a non-zero bit is found.
- *
- * Conversion only fails when the input is too large or is not a
+ /* Conversion only fails when the input is too large or is not a
* whole number, never because of lack of precision because
- * 64-bit integers always have more precision than the 52-bits
- * of a double.
+ * 64-bit integers always have more precision than the 23 bits
+ * of a single.
*/
- nNonZeroBitsCount = IEEE754_Private_CountNonZeroBits(SINGLE_NUM_SIGNIFICAND_BITS, uSingleleSignificand);
+ nPrecisionBits = IEEE754_Private_CountPrecisionBits(uSingleleSignificand) -
+ (64 - SINGLE_NUM_SIGNIFICAND_BITS);
- if(nNonZeroBitsCount && nNonZeroBitsCount > nSingleUnbiasedExponent) {
+ if(nPrecisionBits && nPrecisionBits > nSingleUnbiasedExponent) {
/* --- Not a whole number --- */
Result.type = IEEE754_ToInt_NO_CONVERSION;
} else {
@@ -820,6 +827,48 @@
return Result;
}
+
+
+/* Public function; see ieee754.h */
+double
+IEEE754_UintToDouble(const uint64_t uInt, const int uIsNegative)
+{
+ int nDoubleUnbiasedExponent;
+ uint64_t uDoubleSignificand;
+ int nPrecisionBits;
+
+ /* Figure out the exponent and normalize the significand. This is
+ * done by shifting out all leading zero bits and counting them. If
+ * none are shifted out, the exponent is 63. */
+ uDoubleSignificand = uInt;
+ nDoubleUnbiasedExponent = 63;
+ while(1) {
+ if(uDoubleSignificand & 0x8000000000000000UL) {
+ break;
+ }
+ uDoubleSignificand <<= 1;
+ nDoubleUnbiasedExponent--;
+ };
+
+ /* Position significand correctly for a double. Only shift 63 bits
+ * because of the 1 that is present by implication in IEEE 754.*/
+ uDoubleSignificand >>= 63 - DOUBLE_NUM_SIGNIFICAND_BITS;
+
+ /* Subtract 1 which is present by implication in IEEE 754 */
+ uDoubleSignificand -= 1ULL << (DOUBLE_NUM_SIGNIFICAND_BITS);
+
+ nPrecisionBits = IEEE754_Private_CountPrecisionBits(uInt) - (64 - nDoubleUnbiasedExponent);
+
+ if(nPrecisionBits > DOUBLE_NUM_SIGNIFICAND_BITS) {
+ /* Will lose precision if converted */
+ return IEEE754_UINT_TO_DOUBLE_OOB;
+ }
+
+ return IEEE754_AssembleDouble((uint64_t)uIsNegative,
+ uDoubleSignificand,
+ nDoubleUnbiasedExponent);
+}
+
#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */
diff --git a/src/ieee754.h b/src/ieee754.h
index 53ab3eb..c893e6f 100644
--- a/src/ieee754.h
+++ b/src/ieee754.h
@@ -184,6 +184,25 @@
struct IEEE754_ToInt
IEEE754_SingleToInt(float f);
+
+/**
+ * @brief Convert an unsigned integer to a double with no precision loss.
+ *
+ * @param[in] uInt The value to convert.
+ * @param[in] uIsNegative 0 if postive, 1 if negative.
+ *
+ * @returns Either the converted number or 0.5 if no conversion.
+ *
+ * The conversion will fail if the input can not be represented in the
+ * 52 bits or precision that a double has. 0.5 is returned to indicate
+ * no conversion. It is out-of-band from non-error results, because
+ * all non-error results are whole integers.
+ */
+#define IEEE754_UINT_TO_DOUBLE_OOB 0.5
+double
+IEEE754_UintToDouble(uint64_t uInt, int uIsNegative);
+
+
#endif /* ! QCBOR_DISABLE_PREFERRED_FLOAT */
diff --git a/src/qcbor_decode.c b/src/qcbor_decode.c
index bb8d4d4..64818df 100644
--- a/src/qcbor_decode.c
+++ b/src/qcbor_decode.c
@@ -7042,3 +7042,92 @@
}
#endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */
+
+
+#if !defined(USEFULBUF_DISABLE_ALL_FLOAT) && !defined(QCBOR_DISABLE_PREFERRED_FLOAT)
+/*
+ * Public function, see header qcbor/qcbor_spiffy_decode.h file
+ */
+void
+QCBORDecode_GetNumberConvertPrecisely(QCBORDecodeContext *pMe,
+ QCBORItem *pNumber)
+{
+ QCBORItem Item;
+ struct IEEE754_ToInt ToInt;
+ double d;
+ QCBORError uError;
+
+ if(pMe->uLastError != QCBOR_SUCCESS) {
+ return;
+ }
+
+ uError = QCBORDecode_GetNext(pMe, &Item);
+ if(uError != QCBOR_SUCCESS) {
+ pMe->uLastError = (uint8_t)uError;
+ return;
+ }
+
+ switch(Item.uDataType) {
+ case QCBOR_TYPE_INT64:
+ case QCBOR_TYPE_UINT64:
+ *pNumber = Item;
+ break;
+
+ case QCBOR_TYPE_DOUBLE:
+ ToInt = IEEE754_DoubleToInt(Item.val.dfnum);
+ if(ToInt.type == IEEE754_ToInt_IS_INT) {
+ pNumber->uDataType = QCBOR_TYPE_INT64;
+ pNumber->val.int64 = ToInt.integer.is_signed;
+ } else if(ToInt.type == IEEE754_ToInt_IS_UINT) {
+ if(ToInt.integer.un_signed <= INT64_MAX) {
+ /* Do the same as base QCBOR integer decoding */
+ pNumber->uDataType = QCBOR_TYPE_INT64;
+ pNumber->val.int64 = (int64_t)ToInt.integer.un_signed;
+ } else {
+ pNumber->uDataType = QCBOR_TYPE_UINT64;
+ pNumber->val.uint64 = ToInt.integer.un_signed;
+ }
+ } else {
+ *pNumber = Item;
+ }
+ break;
+
+ case QCBOR_TYPE_FLOAT:
+ ToInt = IEEE754_SingleToInt(Item.val.fnum);
+ if(ToInt.type == IEEE754_ToInt_IS_INT) {
+ pNumber->uDataType = QCBOR_TYPE_INT64;
+ pNumber->val.int64 = ToInt.integer.is_signed;
+ } else if(ToInt.type == IEEE754_ToInt_IS_UINT) {
+ if(ToInt.integer.un_signed <= INT64_MAX) {
+ /* Do the same as base QCBOR integer decoding */
+ pNumber->uDataType = QCBOR_TYPE_INT64;
+ pNumber->val.int64 = (int64_t)ToInt.integer.un_signed;
+ } else {
+ pNumber->uDataType = QCBOR_TYPE_UINT64;
+ pNumber->val.uint64 = ToInt.integer.un_signed;
+ }
+ } else {
+ *pNumber = Item;
+ }
+ break;
+
+
+ case QCBOR_TYPE_65BIT_NEG_INT:
+ d = IEEE754_UintToDouble(Item.val.uint64, 1);
+ if(d == IEEE754_UINT_TO_DOUBLE_OOB) {
+ *pNumber = Item;
+ } else {
+ pNumber->uDataType = QCBOR_TYPE_DOUBLE;
+ /* -1 is because of CBOR offset of negative numbers */
+ pNumber->val.dfnum = d - 1;
+ }
+ break;
+
+ default:
+ pMe->uLastError = QCBOR_ERR_UNEXPECTED_TYPE;
+ pNumber->uDataType = QCBOR_TYPE_NONE;
+ break;
+ }
+}
+
+#endif /* ! USEFULBUF_DISABLE_ALL_FLOAT && ! QCBOR_DISABLE_PREFERRED_FLOAT */
diff --git a/test/qcbor_decode_tests.c b/test/qcbor_decode_tests.c
index dc5d070..833b5b8 100644
--- a/test/qcbor_decode_tests.c
+++ b/test/qcbor_decode_tests.c
@@ -8832,6 +8832,213 @@
}
+#if !defined(USEFULBUF_DISABLE_ALL_FLOAT) && !defined(QCBOR_DISABLE_PREFERRED_FLOAT)
+
+struct PreciseNumberConversion {
+ char *szDescription;
+ UsefulBufC CBOR;
+ QCBORError uError;
+ uint8_t qcborType;
+ struct {
+ int64_t int64;
+ uint64_t uint64;
+ double d;
+ } number;
+};
+
+
+static const struct PreciseNumberConversion PreciseNumberConversions[] = {
+ {
+ "-0.00",
+ {"\xf9\x80\x00", 3},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_INT64,
+ {0, 0, 0}
+ },
+ {
+ "NaN",
+ {"\xf9\x7e\x00", 3},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, NAN}
+ },
+ {
+ "NaN payload",
+ {"\xFB\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, NAN}
+ },
+ {
+ "65536.0 single",
+ {"\xFA\x47\x80\x00\x00", 5},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_INT64,
+ {65536, 0, 0}
+ },
+ {
+ "Infinity",
+ {"\xf9\x7c\x00", 3},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, INFINITY}
+ },
+ {
+ "1.0",
+ {"\xf9\x3c\x00", 3},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_INT64,
+ {1, 0, 0}
+ },
+ {
+ "UINT64_MAX",
+ {"\x1B\xff\xff\xff\xff\xff\xff\xff\xff", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_UINT64,
+ {0, UINT64_MAX, 0}
+ },
+ {
+ "INT64_MIN",
+ {"\x3B\x7f\xff\xff\xff\xff\xff\xff\xff", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_INT64,
+ {INT64_MIN, 0, 0}
+ },
+ {
+ "18446742974197923840",
+ {"\xFB\x43\xEF\xFF\xFF\xE0\x00\x00\x00", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_UINT64,
+ {0, 18446742974197923840ULL, 0}
+ },
+ {
+ "65-bit neg, too much precision",
+ {"\x3B\x80\x00\x00\x00\x00\x00\x00\x01", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_65BIT_NEG_INT,
+ {0, 0x8000000000000001, 0}
+ },
+ {
+ "65-bit neg lots of precision",
+ {"\x3B\xff\xff\xff\xff\xff\xff\xf0\x00", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, -18446744073709547521.0}
+ },
+ {
+ "65-bit neg very precise",
+ {"\x3B\xff\xff\xff\xff\xff\xff\xf8\x00", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, -18446744073709549569.0}
+ },
+ {
+ "65-bit neg too precise",
+ {"\x3B\xff\xff\xff\xff\xff\xff\xfc\x00", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_65BIT_NEG_INT,
+ {0, 18446744073709550592ULL, 0.0}
+ },
+ {
+ "65-bit neg, power of two",
+ {"\x3B\x80\x00\x00\x00\x00\x00\x00\x00", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, -9223372036854775809.0}
+ },
+ {
+ "Zero",
+ {"\x00", 1},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_INT64,
+ {0, 0, 0}
+ },
+ {
+ "Pi",
+ {"\xFB\x40\x09\x2A\xDB\x40\x2D\x16\xB9", 9},
+ QCBOR_SUCCESS,
+ QCBOR_TYPE_DOUBLE,
+ {0, 0, 3.145926}
+ },
+ {
+ "String",
+ {"\x60", 1},
+ QCBOR_ERR_UNEXPECTED_TYPE,
+ QCBOR_TYPE_NONE,
+ {0, 0, 0}
+ }
+};
+
+
+int32_t
+PreciseNumbersTest(void)
+{
+ int i;
+ QCBORError uErr;
+ QCBORItem Item;
+ QCBORDecodeContext DCtx;
+ const struct PreciseNumberConversion *pTest;
+
+ const int count = (int)C_ARRAY_COUNT(PreciseNumberConversions, struct PreciseNumberConversion);
+
+ for(i = 0; i < count; i++) {
+ pTest = &PreciseNumberConversions[i];
+
+ if(i == 11) {
+ uErr = 99; // For break point only
+ }
+
+ QCBORDecode_Init(&DCtx, pTest->CBOR, 0);
+
+ QCBORDecode_GetNumberConvertPrecisely(&DCtx, &Item);
+
+ uErr = QCBORDecode_GetError(&DCtx);
+
+ if(uErr != pTest->uError) {
+ return i * 1000 + (int)uErr;
+ }
+
+ if(pTest->qcborType != Item.uDataType) {
+ return i * 1000 + 200;
+ }
+
+ if(pTest->qcborType == QCBOR_TYPE_NONE) {
+ continue;
+ }
+
+ switch(pTest->qcborType) {
+ case QCBOR_TYPE_INT64:
+ if(Item.val.int64 != pTest->number.int64) {
+ return i * 1000 + 300;
+ }
+ break;
+
+ case QCBOR_TYPE_UINT64:
+ case QCBOR_TYPE_65BIT_NEG_INT:
+ if(Item.val.uint64 != pTest->number.uint64) {
+ return i * 1000 + 400;
+ }
+ break;
+
+ case QCBOR_TYPE_DOUBLE:
+ if(isnan(pTest->number.d)) {
+ if(!isnan(Item.val.dfnum)) {
+ return i * 1000 + 600;
+ }
+ } else {
+ if(Item.val.dfnum != pTest->number.d) {
+ return i * 1000 + 500;
+ }
+ }
+ break;
+ }
+ }
+ return 0;
+}
+
+#endif /* ! USEFULBUF_DISABLE_ALL_FLOAT && ! QCBOR_DISABLE_PREFERRED_FLOAT */
+
+
int32_t
ErrorHandlingTests(void)
{
diff --git a/test/qcbor_decode_tests.h b/test/qcbor_decode_tests.h
index 0cedf43..bd0996b 100644
--- a/test/qcbor_decode_tests.h
+++ b/test/qcbor_decode_tests.h
@@ -319,6 +319,7 @@
int32_t CBORTestIssue134(void);
+int32_t PreciseNumbersTest(void);
int32_t ErrorHandlingTests(void);
diff --git a/test/run_tests.c b/test/run_tests.c
index c61fcef..d5948f2 100644
--- a/test/run_tests.c
+++ b/test/run_tests.c
@@ -124,6 +124,7 @@
#ifndef QCBOR_DISABLE_PREFERRED_FLOAT
TEST_ENTRY(HalfPrecisionAgainstRFCCodeTest),
TEST_ENTRY(FloatValuesTests),
+ TEST_ENTRY(PreciseNumbersTest),
#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */
TEST_ENTRY(GeneralFloatEncodeTests),
TEST_ENTRY(GeneralFloatDecodeTests),