mirror/JUCE
1
0
Fork 0
mirror of https://github.com/juce-framework/JUCE.git synced 2026-01-10 23:44:24 +00:00
Code Activity

Text: Improve string validation

Browse source 
- Reject surrogate code points for all unicode encodings
- Prevent out of bounds access in some cases
- Move ASCII and UTF character validation functions to CharacterFunctions
- Add more unit tests
This commit is contained in:
Anthony Nicholls 2024-07-19 15:33:49 +01:00
parent 1dd2b3c90e
commit 0a78fefecb
37 changed files with 723 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

6
examples/DemoRunner/Builds/Android/app/CMakeLists.txt
View file

@ -1211,8 +1211,11 @@ add_library( ${BINARY_NAME}
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"
@ -3732,8 +3735,11 @@ set_source_files_properties(
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"

9
examples/DemoRunner/Builds/VisualStudio2019/DemoRunner_App.vcxproj
View file

@ -1538,6 +1538,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
examples/DemoRunner/Builds/VisualStudio2019/DemoRunner_App.vcxproj.filters
View file

@ -2272,6 +2272,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
examples/DemoRunner/Builds/VisualStudio2022/DemoRunner_App.vcxproj
View file

@ -1538,6 +1538,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
examples/DemoRunner/Builds/VisualStudio2022/DemoRunner_App.vcxproj.filters
View file

@ -2272,6 +2272,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

6
extras/AudioPerformanceTest/Builds/Android/app/CMakeLists.txt
View file

@ -1070,8 +1070,11 @@ add_library( ${BINARY_NAME}
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"
@ -3274,8 +3277,11 @@ set_source_files_properties(
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"

9
extras/AudioPerformanceTest/Builds/VisualStudio2022/AudioPerformanceTest_App.vcxproj
View file

@ -1357,6 +1357,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/AudioPerformanceTest/Builds/VisualStudio2022/AudioPerformanceTest_App.vcxproj.filters
View file

@ -1948,6 +1948,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

6
extras/AudioPluginHost/Builds/Android/app/CMakeLists.txt
View file

@ -1103,8 +1103,11 @@ add_library( ${BINARY_NAME}
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"
@ -3460,8 +3463,11 @@ set_source_files_properties(
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"

9
extras/AudioPluginHost/Builds/VisualStudio2019/AudioPluginHost_App.vcxproj
View file

@ -1365,6 +1365,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/AudioPluginHost/Builds/VisualStudio2019/AudioPluginHost_App.vcxproj.filters
View file

@ -2023,6 +2023,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/AudioPluginHost/Builds/VisualStudio2022/AudioPluginHost_App.vcxproj
View file

@ -1365,6 +1365,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/AudioPluginHost/Builds/VisualStudio2022/AudioPluginHost_App.vcxproj.filters
View file

@ -2023,6 +2023,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/BinaryBuilder/Builds/VisualStudio2022/BinaryBuilder_ConsoleApp.vcxproj
View file

@ -391,6 +391,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/BinaryBuilder/Builds/VisualStudio2022/BinaryBuilder_ConsoleApp.vcxproj.filters
View file

@ -373,6 +373,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

6
extras/NetworkGraphicsDemo/Builds/Android/app/CMakeLists.txt
View file

@ -1074,8 +1074,11 @@ add_library( ${BINARY_NAME}
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"
@ -3358,8 +3361,11 @@ set_source_files_properties(
"../../../../../modules/juce_core/text/juce_CharacterFunctions.h"
"../../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF8_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF16_test.cpp"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32.h"
"../../../../../modules/juce_core/text/juce_CharPointer_UTF32_test.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.cpp"
"../../../../../modules/juce_core/text/juce_Identifier.h"
"../../../../../modules/juce_core/text/juce_LocalisedStrings.cpp"

9
extras/NetworkGraphicsDemo/Builds/VisualStudio2022/NetworkGraphicsDemo_App.vcxproj
View file

@ -1357,6 +1357,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/NetworkGraphicsDemo/Builds/VisualStudio2022/NetworkGraphicsDemo_App.vcxproj.filters
View file

@ -1978,6 +1978,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/Projucer/Builds/VisualStudio2019/Projucer_App.vcxproj
View file

@ -508,6 +508,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/Projucer/Builds/VisualStudio2019/Projucer_App.vcxproj.filters
View file

@ -826,6 +826,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/Projucer/Builds/VisualStudio2022/Projucer_App.vcxproj
View file

@ -508,6 +508,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/Projucer/Builds/VisualStudio2022/Projucer_App.vcxproj.filters
View file

@ -826,6 +826,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/UnitTestRunner/Builds/VisualStudio2019/UnitTestRunner_ConsoleApp.vcxproj
View file

@ -1373,6 +1373,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/UnitTestRunner/Builds/VisualStudio2019/UnitTestRunner_ConsoleApp.vcxproj.filters
View file

@ -2044,6 +2044,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/UnitTestRunner/Builds/VisualStudio2022/UnitTestRunner_ConsoleApp.vcxproj
View file

@ -1373,6 +1373,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/UnitTestRunner/Builds/VisualStudio2022/UnitTestRunner_ConsoleApp.vcxproj.filters
View file

@ -2044,6 +2044,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

9
extras/WindowsDLL/Builds/VisualStudio2022/WindowsDLL_StaticLibrary.vcxproj
View file

@ -1356,6 +1356,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>

9
extras/WindowsDLL/Builds/VisualStudio2022/WindowsDLL_StaticLibrary.vcxproj.filters
View file

@ -1975,6 +1975,15 @@
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharacterFunctions.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF8_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF16_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_CharPointer_UTF32_test.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\modules\juce_core\text\juce_Identifier.cpp">
<Filter>JUCE Modules\juce_core\text</Filter>
</ClCompile>

3
modules/juce_core/juce_core.cpp
View file

@ -295,6 +295,9 @@
#include "containers/juce_FixedSizeFunction_test.cpp"
#include "javascript/juce_JSONSerialisation_test.cpp"
#include "memory/juce_SharedResourcePointer_test.cpp"
#include "text/juce_CharPointer_UTF8_test.cpp"
#include "text/juce_CharPointer_UTF16_test.cpp"
#include "text/juce_CharPointer_UTF32_test.cpp"
#if JUCE_MAC || JUCE_IOS
#include "native/juce_ObjCHelpers_mac_test.mm"
#endif

2
modules/juce_core/text/juce_CharPointer_ASCII.h
View file

@ -366,7 +366,7 @@ public:
/** Returns true if the given unicode character can be represented in this encoding. */
static bool canRepresent (juce_wchar character) noexcept
{
return ((unsigned int) character) < (unsigned int) 128;
return CharacterFunctions::isAscii (character);
}
/** Returns true if this data contains a valid string in this encoding. */

41
modules/juce_core/text/juce_CharPointer_UTF16.h
View file

@ -438,36 +438,33 @@ public:
/** Returns true if the given unicode character can be represented in this encoding. */
static bool canRepresent (juce_wchar character) noexcept
{
auto n = (uint32) character;
return n < 0x10ffff && (n < 0xd800 || n > 0xdfff);
return CharacterFunctions::isNonSurrogateCodePoint (character);
}
/** Returns true if this data contains a valid string in this encoding. */
static bool isValidString (const CharType* dataToTest, int maxBytesToRead)
static bool isValidString (const CharType* codeUnits, int maxBytesToRead)
{
maxBytesToRead /= (int) sizeof (CharType);
const auto maxCodeUnitsToRead = (size_t) maxBytesToRead / sizeof (CharType);
while (--maxBytesToRead >= 0 && *dataToTest != 0)
for (size_t codeUnitIndex = 0; codeUnitIndex < maxCodeUnitsToRead; ++codeUnitIndex)
{
auto n = (uint32) (uint16) *dataToTest++;
const auto c = toCodePoint (codeUnits[codeUnitIndex]);
if (n >= 0xd800)
{
if (n > 0x10ffff)
if (c == 0)
return true;
if (canRepresent (c))
continue;
if (! CharacterFunctions::isHighSurrogate (c))
return false;
if (n <= 0xdfff)
{
if (n > 0xdc00)
if (++codeUnitIndex >= maxCodeUnitsToRead)
return false;
auto nextChar = (uint32) (uint16) *dataToTest++;
if (nextChar < 0xdc00 || nextChar > 0xdfff)
if (! CharacterFunctions::isLowSurrogate (toCodePoint (codeUnits[codeUnitIndex])))
return false;
}
}
}
return true;
}
@ -527,6 +524,16 @@ private:
return n;
}
static inline uint32 toUint32 (CharType c) noexcept
{
return (uint32) (uint16) c;
}
static inline juce_wchar toCodePoint (CharType c) noexcept
{
return (juce_wchar) toUint32 (c);
}
};
} // namespace juce

118
modules/juce_core/text/juce_CharPointer_UTF16_test.cpp Normal file
View file

@ -0,0 +1,118 @@
/*
==============================================================================
This file is part of the JUCE framework.
Copyright (c) Raw Material Software Limited
JUCE is an open source framework subject to commercial or open source
licensing.
By downloading, installing, or using the JUCE framework, or combining the
JUCE framework with any other source code, object code, content or any other
copyrightable work, you agree to the terms of the JUCE End User Licence
Agreement, and all incorporated terms including the JUCE Privacy Policy and
the JUCE Website Terms of Service, as applicable, which will bind you. If you
do not agree to the terms of these agreements, we will not license the JUCE
framework to you, and you must discontinue the installation or download
process and cease use of the JUCE framework.
JUCE End User Licence Agreement: https://juce.com/legal/juce-8-licence/
JUCE Privacy Policy: https://juce.com/juce-privacy-policy
JUCE Website Terms of Service: https://juce.com/juce-website-terms-of-service/
Or:
You may also use this code under the terms of the AGPLv3:
https://www.gnu.org/licenses/agpl-3.0.en.html
THE JUCE FRAMEWORK IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL
WARRANTIES, WHETHER EXPRESSED OR IMPLIED, INCLUDING WARRANTY OF
MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, ARE DISCLAIMED.
==============================================================================
*/
namespace juce
{
class CharPointer_UTF16Test final : public UnitTest
{
public:
CharPointer_UTF16Test() : UnitTest { "CharPointer_UTF16Test", UnitTestCategories::text } {}
void runTest() final
{
const auto toCharType = [] (const std::vector<char16_t>& str)
{
return reinterpret_cast<const CharPointer_UTF16::CharType*> (str.data());
};
const auto getNumBytes = [] (const auto& str)
{
return (int) (sizeof (CharPointer_UTF16::CharType) * str.size());
};
beginTest ("String validation - empty string / null-terminator");
{
const std::vector<CharPointer_UTF16::CharType> string { 0x0 };
expect (CharPointer_UTF16::isValidString (string.data(), getNumBytes (string)));
}
beginTest ("String validation - ascii");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0x21, 0x0 }; // Test!
expect (CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - 2-byte code points");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0x20ac, 0x0 }; // Test€
expect (CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - surrogate pairs");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0xd83d, 0xde03, 0x0 }; // Test😃
expect (CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - high-surrogate without a low-surrogate");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0xd83d, 0x0 };
expect (! CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - low-surrogate without a high-surrogate");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0xde03, 0x0 };
expect (! CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - characters after a null terminator are ignored");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0x0, 0xde03 };
expect (CharPointer_UTF16::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - characters exceeding max bytes are ignored");
{
const std::vector<char16_t> string { 0x54, 0x65, 0x73, 0x74, 0xde03 };
expect (CharPointer_UTF16::isValidString (toCharType (string), 8));
}
beginTest ("String validation - all unicode characters");
{
for (uint32_t c = 0; c < 0x110000; ++c)
{
std::array<CharPointer_UTF16::CharType, 2> string = {};
CharPointer_UTF16 utf16 { string.data() };
utf16.write ((juce_wchar) c);
expect (CharPointer_UTF16::isValidString (string.data(), 4) == CharPointer_UTF32::canRepresent ((juce_wchar) c));
}
}
}
};
static CharPointer_UTF16Test charPointer_UTF16Test;
} // namespace juce

17
modules/juce_core/text/juce_CharPointer_UTF32.h
View file

@ -357,17 +357,24 @@ public:
/** Returns true if the given unicode character can be represented in this encoding. */
static bool canRepresent (juce_wchar character) noexcept
{
return ((uint32) character) < (uint32) 0x10ffff;
return CharacterFunctions::isNonSurrogateCodePoint (character);
}
/** Returns true if this data contains a valid string in this encoding. */
static bool isValidString (const CharType* dataToTest, int maxBytesToRead)
static bool isValidString (const CharType* codeUnits, int maxBytesToRead)
{
maxBytesToRead /= (int) sizeof (CharType);
const auto maxCodeUnitsToRead = (size_t) maxBytesToRead / sizeof (CharType);
while (--maxBytesToRead >= 0 && *dataToTest != 0)
if (! canRepresent (*dataToTest++))
for (size_t codeUnitIndex = 0; codeUnitIndex < maxCodeUnitsToRead; ++codeUnitIndex)
{
const auto c = codeUnits[codeUnitIndex];
if (c == 0)
return true;
if (! canRepresent (c))
return false;
}
return true;
}

108
modules/juce_core/text/juce_CharPointer_UTF32_test.cpp Normal file
View file

@ -0,0 +1,108 @@
/*
==============================================================================
This file is part of the JUCE framework.
Copyright (c) Raw Material Software Limited
JUCE is an open source framework subject to commercial or open source
licensing.
By downloading, installing, or using the JUCE framework, or combining the
JUCE framework with any other source code, object code, content or any other
copyrightable work, you agree to the terms of the JUCE End User Licence
Agreement, and all incorporated terms including the JUCE Privacy Policy and
the JUCE Website Terms of Service, as applicable, which will bind you. If you
do not agree to the terms of these agreements, we will not license the JUCE
framework to you, and you must discontinue the installation or download
process and cease use of the JUCE framework.
JUCE End User Licence Agreement: https://juce.com/legal/juce-8-licence/
JUCE Privacy Policy: https://juce.com/juce-privacy-policy
JUCE Website Terms of Service: https://juce.com/juce-website-terms-of-service/
Or:
You may also use this code under the terms of the AGPLv3:
https://www.gnu.org/licenses/agpl-3.0.en.html
THE JUCE FRAMEWORK IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL
WARRANTIES, WHETHER EXPRESSED OR IMPLIED, INCLUDING WARRANTY OF
MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, ARE DISCLAIMED.
==============================================================================
*/
namespace juce
{
class CharPointer_UTF32Test final : public UnitTest
{
public:
CharPointer_UTF32Test() : UnitTest { "CharPointer_UTF32", UnitTestCategories::text } {}
void runTest() final
{
const auto toCharType = [] (const std::vector<char32_t>& str)
{
return reinterpret_cast<const CharPointer_UTF32::CharType*> (str.data());
};
const auto getNumBytes = [] (const auto& str)
{
return (int) (sizeof (CharPointer_UTF32::CharType) * str.size());
};
beginTest ("String validation - empty string / null-terminator");
{
const std::vector<CharPointer_UTF32::CharType> string { 0x0 };
expect (CharPointer_UTF32::isValidString (string.data(), getNumBytes (string)));
}
beginTest ("String validation - ascii");
{
const std::vector<char32_t> string { 0x54, 0x65, 0x73, 0x74, 0x21, 0x0 }; // Test!
expect (CharPointer_UTF32::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - 2-byte code points");
{
const std::vector<char32_t> string { 0x54, 0x65, 0x73, 0x74, 0x20ac, 0x0 }; // Test€
expect (CharPointer_UTF32::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - maximum code point");
{
const std::vector<char32_t> string1 { 0x54, 0x65, 0x73, 0x74, 0x10ffff, 0x0 };
expect (CharPointer_UTF32::isValidString (toCharType (string1), getNumBytes (string1)));
const std::vector<char32_t> string2 { 0x54, 0x65, 0x73, 0x74, 0x110000, 0x0 };
expect (! CharPointer_UTF32::isValidString (toCharType (string2), getNumBytes (string2)));
}
beginTest ("String validation - characters after a null terminator are ignored");
{
const std::vector<char32_t> string { 0x54, 0x65, 0x73, 0x74, 0x0, 0x110000 };
expect (CharPointer_UTF32::isValidString (toCharType (string), getNumBytes (string)));
}
beginTest ("String validation - characters exceeding max bytes are ignored");
{
const std::vector<char32_t> string { 0x54, 0x65, 0x73, 0x74, 0x110000 };
expect (CharPointer_UTF32::isValidString (toCharType (string), 8));
}
beginTest ("String validation - surrogate code points are invalid");
{
const std::vector<char32_t> highSurrogate { 0xd800 };
expect (! CharPointer_UTF32::isValidString (toCharType (highSurrogate), getNumBytes (highSurrogate)));
const std::vector<char32_t> lowSurrogate { 0xdfff };
expect (! CharPointer_UTF32::isValidString (toCharType (lowSurrogate), getNumBytes (lowSurrogate)));
}
}
};
static CharPointer_UTF32Test charPointer_UTF32Test;
} // namespace juce

75
modules/juce_core/text/juce_CharPointer_UTF8.h
View file

@ -497,45 +497,80 @@ public:
/** Returns true if the given unicode character can be represented in this encoding. */
static bool canRepresent (juce_wchar character) noexcept
{
return ((uint32) character) < (uint32) 0x10ffff;
return CharacterFunctions::isNonSurrogateCodePoint (character);
}
/** Returns true if this data contains a valid string in this encoding. */
static bool isValidString (const CharType* dataToTest, int maxBytesToRead)
static bool isValidString (const CharType* codeUnits, int maxBytesToRead)
{
while (--maxBytesToRead >= 0 && *dataToTest != 0)
{
auto byte = (signed char) *dataToTest++;
const auto maxCodeUnitsToRead = (size_t) maxBytesToRead / sizeof (CharType);
if (byte < 0)
for (size_t codeUnitIndex = 0; codeUnitIndex < maxCodeUnitsToRead; ++codeUnitIndex)
{
int bit = 0x40;
int numExtraValues = 0;
const auto firstByte = (uint8_t) codeUnits[codeUnitIndex];
while ((byte & bit) != 0)
if (firstByte == 0)
return true;
if (CharacterFunctions::isAscii ((juce_wchar) firstByte))
continue;
auto numExtraBytes = [&]
{
if (bit < 8)
if (firstByte < 0xc0)
return 0;
if (firstByte < 0xe0)
return 1;
if (firstByte < 0xf0)
return 2;
if (firstByte <= 0xf4)
return 3;
return 0;
}();
if (numExtraBytes == 0)
return false;
++numExtraValues;
bit >>= 1;
auto bytes = (uint32_t) firstByte;
if (bit == 8 && (numExtraValues > maxBytesToRead
|| *CharPointer_UTF8 (dataToTest - 1) > 0x10ffff))
while (numExtraBytes--)
{
if (++codeUnitIndex >= maxCodeUnitsToRead)
return false;
bytes <<= 8;
bytes |= (uint32_t) (uint8_t) codeUnits[codeUnitIndex];
}
if (numExtraValues == 0)
if (constexpr uint32_t firstTwoByteCodePoint = 0xc280; bytes < firstTwoByteCodePoint)
return false;
maxBytesToRead -= numExtraValues;
if (maxBytesToRead < 0)
if (constexpr uint32_t lastTwoByteCodePoint = 0xdfbf; bytes <= lastTwoByteCodePoint)
continue;
if (constexpr uint32_t firstThreeByteCodePoint = 0xe0a080; bytes < firstThreeByteCodePoint)
return false;
while (--numExtraValues >= 0)
if ((*dataToTest++ & 0xc0) != 0x80)
if (constexpr uint32_t firstSurrogateCodePoint = 0xeda080; bytes < firstSurrogateCodePoint)
continue;
if (constexpr uint32_t lastSurrogateCodePoint = 0xedbfbf; bytes <= lastSurrogateCodePoint)
return false;
if (constexpr uint32_t lastThreeByteCodePoint = 0xefbfbf; bytes <= lastThreeByteCodePoint)
continue;
if (constexpr uint32_t firstFourByteCodePoint = 0xf0908080; bytes < firstFourByteCodePoint)
return false;
if (constexpr uint32_t lastFourByteCodePoint = 0xf48fbfbf; bytes <= lastFourByteCodePoint)
continue;
return false;
}
}
return true;

91
modules/juce_core/text/juce_CharPointer_UTF8_test.cpp Normal file
View file

@ -0,0 +1,91 @@
/*
==============================================================================
This file is part of the JUCE framework.
Copyright (c) Raw Material Software Limited
JUCE is an open source framework subject to commercial or open source
licensing.
By downloading, installing, or using the JUCE framework, or combining the
JUCE framework with any other source code, object code, content or any other
copyrightable work, you agree to the terms of the JUCE End User Licence
Agreement, and all incorporated terms including the JUCE Privacy Policy and
the JUCE Website Terms of Service, as applicable, which will bind you. If you
do not agree to the terms of these agreements, we will not license the JUCE
framework to you, and you must discontinue the installation or download
process and cease use of the JUCE framework.
JUCE End User Licence Agreement: https://juce.com/legal/juce-8-licence/
JUCE Privacy Policy: https://juce.com/juce-privacy-policy
JUCE Website Terms of Service: https://juce.com/juce-website-terms-of-service/
Or:
You may also use this code under the terms of the AGPLv3:
https://www.gnu.org/licenses/agpl-3.0.en.html
THE JUCE FRAMEWORK IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL
WARRANTIES, WHETHER EXPRESSED OR IMPLIED, INCLUDING WARRANTY OF
MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, ARE DISCLAIMED.
==============================================================================
*/
namespace juce
{
class CharPointer_UTF8Test final : public UnitTest
{
public:
CharPointer_UTF8Test() : UnitTest { "CharPointer_UTF8", UnitTestCategories::text } {}
void runTest() final
{
beginTest ("String validation - empty string / null-terminator");
{
const std::vector<CharPointer_UTF8::CharType> string { 0x0 };
expect (CharPointer_UTF8::isValidString (string.data(), (int) string.size()));
}
beginTest ("String validation - ascii");
{
const std::vector<CharPointer_UTF8::CharType> string { 0x54, 0x65, 0x73, 0x74, 0x21, 0x0 }; // Test!
expect (CharPointer_UTF8::isValidString (string.data(), (int) string.size()));
}
beginTest ("String validation - continuation characters are invalid when not proceeded by the correct bytes");
{
const std::vector<CharPointer_UTF8::CharType> string { -1 };
expect (! CharPointer_UTF8::isValidString (string.data(), (int) string.size()));
}
beginTest ("String validation - characters after a null terminator are ignored");
{
const std::vector<CharPointer_UTF8::CharType> string { 0x54, 0x65, 0x73, 0x74, 0x0, -1 };
expect (CharPointer_UTF8::isValidString (string.data(), (int) string.size()));
}
beginTest ("String validation - characters exceeding max bytes are ignored");
{
const std::vector<CharPointer_UTF8::CharType> string { 0x54, 0x65, 0x73, 0x74, -1 };
expect (CharPointer_UTF8::isValidString (string.data(), 4));
}
beginTest ("String validation - all unicode characters");
{
for (uint32_t c = 0; c < 0x110000; ++c)
{
std::array<CharPointer_UTF8::CharType, 4> string = {};
CharPointer_UTF8 utf8 { string.data() };
utf8.write ((juce_wchar) c);
expect (CharPointer_UTF8::isValidString (string.data(), (int) string.size()) == CharPointer_UTF32::canRepresent ((juce_wchar) c));
}
}
}
};
static CharPointer_UTF8Test charPointer_UTF8Test;
} // namespace juce

64
modules/juce_core/text/juce_CharacterFunctions.h
View file

@ -150,6 +150,70 @@ public:
/** Converts a byte of Windows 1252 codepage to unicode. */
static juce_wchar getUnicodeCharFromWindows1252Codepage (uint8 windows1252Char) noexcept;
/** Returns true if a unicode code point is part of the basic multilingual plane.
@see isAscii, isNonSurrogateCodePoint
*/
static constexpr bool isPartOfBasicMultilingualPlane (juce_wchar character) noexcept
{
return (uint32) character < 0x10000;
}
/** Returns true if a unicode code point is in the range os ASCII characters.
@see isAsciiControlCharacter, isPartOfBasicMultilingualPlane
*/
static constexpr bool isAscii (juce_wchar character) noexcept
{
return (uint32) character < 128;
}
/** Returns true if a unicode code point is in the range of ASCII control characters.
@see isAscii
*/
static constexpr bool isAsciiControlCharacter (juce_wchar character) noexcept
{
return (uint32) character < 32;
}
/** Returns true if a unicode code point is in the range of UTF-16 surrogate code units.
@see isHighSurrogate, isLowSurrogate
*/
static constexpr bool isSurrogate (juce_wchar character) noexcept
{
const auto n = (uint32) character;
return 0xd800 <= n && n <= 0xdfff;
}
/** Returns true if a unicode code point is in the range of UTF-16 high surrogate code units.
@see isLowSurrogate, isSurrogate
*/
static constexpr bool isHighSurrogate (juce_wchar character) noexcept
{
const auto n = (uint32) character;
return 0xd800 <= n && n <= 0xdbff;
}
/** Returns true if a unicode code point is in the range of UTF-16 low surrogate code units.
@see isHighSurrogate, isSurrogate
*/
static constexpr bool isLowSurrogate (juce_wchar character) noexcept
{
const auto n = (uint32) character;
return 0xdc00 <= n && n <= 0xdfff;
}
/** Returns true if a unicode code point is in the range of valid unicode code points. */
static constexpr bool isNonSurrogateCodePoint (juce_wchar character) noexcept
{
const auto n = (uint32) character;
return n <= 0x10ffff && ! isSurrogate (character);
}
//==============================================================================
/** Parses a character string to read a floating-point number.
Note that this will advance the pointer that is passed in, leaving it at