Windows: correctly handle SEH exceptions during self-tests in order to disable CPU extended features in such case.

src/Common/Crypto.c

@@ -1156,20 +1156,20 @@ static BOOL HwEncryptionDisabled = FALSE;
 
 BOOL IsAesHwCpuSupported ()
 {
+#ifdef TC_WINDOWS_BOOT_AES
 	static BOOL state = FALSE;
 	static BOOL stateValid = FALSE;
 
 	if (!stateValid)
 	{
-#ifdef TC_WINDOWS_BOOT_AES
 		state = is_aes_hw_cpu_supported() ? TRUE : FALSE;
-#else
-		state = g_hasAESNI ? TRUE : FALSE;
-#endif
 		stateValid = TRUE;
 	}
 
 	return state && !HwEncryptionDisabled;
+#else
+	return (HasAESNI() && !HwEncryptionDisabled)? TRUE : FALSE;
+#endif
 }
 
 void EnableHwEncryption (BOOL enable)

src/Common/Tests.c

@@ -1355,18 +1355,45 @@ BOOL AutoTestAlgorithms (void)
 {
 	BOOL result = TRUE;
 	BOOL hwEncryptionEnabled = IsHwEncryptionEnabled();
+#if defined (_MSC_VER) && !defined (_UEFI)
+	BOOL exceptionCatched = FALSE;
+	__try
+	{
+#endif
+		EnableHwEncryption (FALSE);
 
-	EnableHwEncryption (FALSE);
+		if (!DoAutoTestAlgorithms())
+			result = FALSE;
 
-	if (!DoAutoTestAlgorithms())
-		result = FALSE;
+		EnableHwEncryption (TRUE);
 
-	EnableHwEncryption (TRUE);
+		if (!DoAutoTestAlgorithms())
+			result = FALSE;
 
-	if (!DoAutoTestAlgorithms())
-		result = FALSE;
+		EnableHwEncryption (hwEncryptionEnabled);
+#if defined (_MSC_VER) && !defined (_UEFI)
+	}
+    __except (EXCEPTION_EXECUTE_HANDLER)
+	{
+		exceptionCatched = TRUE;
+	}
 
-	EnableHwEncryption (hwEncryptionEnabled);
+	if (exceptionCatched)
+	{
+		/* unexepected exception raised. Disable all CPU extended feature and try again */
+		EnableHwEncryption (hwEncryptionEnabled);
+		DisableCPUExtendedFeatures ();
+		__try
+		{
+			result = DoAutoTestAlgorithms();
+		}
+	    __except (EXCEPTION_EXECUTE_HANDLER)
+		{
+			/* exception still occuring. Report failure. */
+			result = FALSE;
+		}
+	}
+#endif
 	return result;
 }
 

src/Crypto/cpu.c

@@ -200,10 +200,10 @@ static uint64 xgetbv()
 #endif
 }
 
-int g_x86DetectionDone = 0;
-int g_hasISSE = 0, g_hasSSE2 = 0, g_hasSSSE3 = 0, g_hasMMX = 0, g_hasAESNI = 0, g_hasCLMUL = 0, g_isP4 = 0;
-int g_hasAVX = 0, g_hasAVX2 = 0, g_hasBMI2 = 0, g_hasSSE42 = 0, g_hasSSE41 = 0, g_isIntel = 0, g_isAMD = 0;
-uint32 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
+volatile int g_x86DetectionDone = 0;
+volatile int g_hasISSE = 0, g_hasSSE2 = 0, g_hasSSSE3 = 0, g_hasMMX = 0, g_hasAESNI = 0, g_hasCLMUL = 0, g_isP4 = 0;
+volatile int g_hasAVX = 0, g_hasAVX2 = 0, g_hasBMI2 = 0, g_hasSSE42 = 0, g_hasSSE41 = 0, g_isIntel = 0, g_isAMD = 0;
+volatile uint32 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
 
 VC_INLINE int IsIntel(const uint32 output[4])
 {
@@ -381,5 +381,23 @@ int is_aes_hw_cpu_supported ()
 	return bHasAESNI;
 }
 
+void DisableCPUExtendedFeatures ()
+{
+	g_hasSSE2 = 0;
+	g_hasISSE = 0;
+	g_hasMMX = 0;
+	g_hasSSE2 = 0;
+	g_hasISSE = 0;
+	g_hasMMX = 0;
+	g_hasAVX = 0;
+	g_hasAVX2 = 0;
+	g_hasBMI2 = 0;
+	g_hasSSE42 = 0;
+	g_hasSSE41 = 0;
+	g_hasSSSE3 = 0;
+	g_hasAESNI = 0;
+	g_hasCLMUL = 0;
+}
+
 #endif
 

src/Crypto/cpu.h

@@ -186,38 +186,31 @@ extern "C" {
 #define CRYPTOPP_CPUID_AVAILABLE
 
 // these should not be used directly
-extern int g_x86DetectionDone;
-extern int g_hasAVX;
-extern int g_hasAVX2;
-extern int g_hasBMI2;
-extern int g_hasSSE42;
-extern int g_hasSSE41;
-extern int g_hasSSSE3;
-extern int g_hasAESNI;
-extern int g_hasCLMUL;
-extern int g_isP4;
-extern int g_isIntel;
-extern int g_isAMD;
-extern uint32 g_cacheLineSize;
+extern volatile int g_x86DetectionDone;
+extern volatile int g_hasSSE2;
+extern volatile int g_hasISSE;
+extern volatile int g_hasMMX;
+extern volatile int g_hasAVX;
+extern volatile int g_hasAVX2;
+extern volatile int g_hasBMI2;
+extern volatile int g_hasSSE42;
+extern volatile int g_hasSSE41;
+extern volatile int g_hasSSSE3;
+extern volatile int g_hasAESNI;
+extern volatile int g_hasCLMUL;
+extern volatile int g_isP4;
+extern volatile int g_isIntel;
+extern volatile int g_isAMD;
+extern volatile uint32 g_cacheLineSize;
 void DetectX86Features(); // must be called at the start of the program/driver
 int CpuId(uint32 input, uint32 *output);
-
-#if CRYPTOPP_BOOL_X64
-#define HasSSE2()	1
-#define HasISSE()	1
-#define HasMMX()	1
-#else
-
-extern int g_hasSSE2;
-extern int g_hasISSE;
-extern int g_hasMMX;
+// disable all CPU extended features (e.g. SSE, AVX, AES) that may have
+// been enabled by DetectX86Features.
+void DisableCPUExtendedFeatures (); 
 
 #define HasSSE2()	g_hasSSE2
 #define HasISSE()	g_hasISSE
 #define HasMMX()	g_hasMMX
-
-#endif
-
 #define HasSSE42() g_hasSSE42
 #define HasSSE41() g_hasSSE41
 #define HasSAVX() g_hasAVX

src/Driver/DumpFilter.c

@@ -66,18 +66,8 @@ NTSTATUS DumpFilterEntry (PFILTER_EXTENSION filterExtension, PFILTER_INITIALIZAT
 	if (filterExtension->DumpType == DumpTypeCrashdump)
 	{
 		dumpConfig.HwEncryptionEnabled = FALSE;
-		// disable also SSE optimizations
-		HasMMX() = 0;
-		HasISSE() = 0;
-		HasSSE2() = 0;
-		HasSSSE3() = 0;
-		HasSSE41() = 0;
-		HasSSE42() = 0;
-		HasAESNI() = 0;
-		HasCLMUL() = 0;
-		HasSAVX() = 0;
-		HasSAVX2() = 0;
-		HasSBMI2() = 0;
+		// disable also CPU extended features used in optimizations
+		DisableCPUExtendedFeatures ();
 	}
 #endif
 

src/Driver/Ntdriver.c

@@ -153,7 +153,16 @@ NTSTATUS DriverEntry (PDRIVER_OBJECT DriverObject, PUNICODE_STRING RegistryPath)
 		if (startKeyValue->Type == REG_DWORD && *((uint32 *) startKeyValue->Data) == SERVICE_BOOT_START)
 		{
 			if (!SelfTestsPassed)
-				TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
+			{
+				// in case of system encryption, if self-tests fail, disable all extended CPU
+				// features and try again in order to workaround faulty configurations
+				DisableCPUExtendedFeatures ();
+				SelfTestsPassed = AutoTestAlgorithms();
+
+				// BUG CHECK if the self-tests still fail
+				if (!SelfTestsPassed)
+					TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
+			}
 
 			LoadBootArguments();
 			VolumeClassFilterRegistered = IsVolumeClassFilterRegistered();