Convert functions to use code points

.gitattributes

@@ -0,0 +1,2 @@
+src/Data/CodePoint/Unicode/Internal.purs linguist-generated
+src/Data/CodePoint/Unicode/Internal/Casing.purs linguist-generated

.github/workflows/ci.yml

@@ -26,28 +26,11 @@ jobs:
             .spago
             output
 
-      - name: Set up Node toolchain
-        uses: actions/setup-node@v1
-        with:
-          node-version: "12.x"
-
-      - name: Cache NPM dependencies
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-node-modules
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
-
-      - name: Install NPM dependencies
-        run: npm install
+      - name: Install dependencies
+        run: spago install
 
-      - name: Build the project
-        run: npm run build
+      - name: Build source
+        run: spago build --no-install --purs-args '--censor-lib --strict'
 
       - name: Run tests
-        run: npm run test
+        run: spago test --no-install

.gitignore

@@ -3,11 +3,16 @@
 !.github
 !.editorconfig
 !.eslintrc.json
+!.gitattributes
 
 output
 generated-docs
 bower_components
 
+UnicodeData.txt
+CaseFolding.txt
+SpecialCasing.txt
+
 node_modules
 package-lock.json
 *.lock

CHANGELOG.md

@@ -6,10 +6,15 @@ Notable changes to this project are documented in this file. The format is based
 
 Breaking changes (😱!!!):
 - Added support for PureScript 0.14 and dropped support for all previous versions (#30 by @JordanMartinez)
+- Main module renamed to `Data.CodePoint.Unicode`, functions now operate on `CodePoint`s, no longer `Char`s (#15 by @MonoidMusician)
+- Simple case conversions renamed `toUpper` -> `toUpperSimple`, etc. (#15 by @MonoidMusician)
+- Deprecation warnings for `isDigit` and `digitToInt` (#31 by @milesfrain)
 
 New features:
 - Added `hexDigitToInt`, `decDigitToInt`, `octDigitToInt` and deprecated `isDigit` and `digitToInt` (#31 by @milesfrain)
-
+- New `toUpper`, `toLower`, `toTitle` based on full Unicode replacements, which may return more than one code point (#15 by @MonoidMusician)
+- Added `caseFold` and `caseFoldSimple` (#15 by @MonoidMusician)
+- New module `Data.String.Unicode` for case-conversion operating on strings (derived from the code point functions) (#15 by @MonoidMusician)
 Bugfixes:
 
 Other improvements:

README.md

@@ -5,7 +5,11 @@
 [![Pursuit](https://pursuit.purescript.org/packages/purescript-unicode/badge)](https://pursuit.purescript.org/packages/purescript-unicode)
 [![Maintainer: cdepillabout](https://img.shields.io/badge/maintainer-cdepillabout-teal.svg)](https://github.com/cdepillabout)
 
-The library summary hasn't been written yet (contributions are welcome!). The library summary describes the library's purpose in one to three sentences.
+A library for working with the properties of Unicode code points, including the general category of a code point, predicates for determining whether a code point is a letter or number character (`isLetter` and `isNumber`, for example), and case conversion functions (`toUpper`, `toLower`, and `toTitle`, as well as `caseFold` for caseless matching).
+
+General functions for working with `String`s in terms of `CodePoint`s are found in the `Data.String.CodePoints` module in the [`strings` library](https://github.com/purescript/purescript-strings).
+
+The version of the [Unicode standard](https://unicode.org/standard/standard.html) supported by this library can be found in the [`unicode-version`](./unicode-version) file.
 
 ## Installation
 

docs/README.md

@@ -4,9 +4,12 @@ This directory contains documentation for `unicode`. If you are interested in co
 
 ## Generating Internal Modules
 
-The [Data.Char.Unicode.Internal](../src/Data/Char/Unicode/Internal.purs) module can be generated with the following command:
+The [Data.CodePoint.Unicode.Internal](../src/Data/CodePoint/Unicode/Internal.purs) and [Data.CodePoint.Unicode.Internal.Casing](../src/Data/CodePoint/Unicode/Internal/Casing.purs) modules can be generated with the following command from the root of this repository:
 
 ```sh
-$ wget 'http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt'
-$ ./ubconfc < UnicodeData.txt > src/Data/Char/Unicode/Internal.purs
-```
+$ ./download.sh
+$ ./ubconfc < UnicodeData.txt > src/Data/CodePoint/Unicode/Internal.purs
+$ ./fullcase.js
+```
+
+(Note that this downloads data according to the version stored in the [`unicode-version`](../unicode-version) file in the root.)

download.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+VERSION=$(cat unicode-version)
+for F in UnicodeData SpecialCasing CaseFolding; do
+  wget "https://www.unicode.org/Public/$VERSION/ucd/$F.txt" -O $F.txt
+done

fullcase.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+const SpecialCasing = ""+fs.readFileSync("SpecialCasing.txt");
+const CaseFolding = ""+fs.readFileSync("CaseFolding.txt");
+
+const special =
+  /^([0-9A-F]+);\s+([0-9A-F]+(?:\s+[0-9A-F]+)*)?;\s+([0-9A-F]+(?:\s+[0-9A-F]+)*)?;\s+([0-9A-F]+(?:\s+[0-9A-F]+)*)?;\s+(#.+?)?$/mg;
+
+const folding =
+  /^([0-9A-F]+);\s+([CFS])?;\s+([0-9A-F]+(?:\s+[0-9A-F]+)*)?;\s+(#.+?)?$/mg;
+
+const scData = {};
+const cfData = {};
+
+const rhex = a => parseInt(a, 16);
+const sortHex = (a,b) => rhex(a) - rhex(b);
+
+for (const spec of SpecialCasing.matchAll(special)) {
+  if (scData[spec[1]]) die("Duplicate special case rule");
+  scData[spec[1]] = spec;
+}
+for (const fold of CaseFolding.matchAll(folding)) {
+  if (cfData[fold[1]]) {
+    cfData[fold[1]][fold[2]] = fold;
+  } else {
+    cfData[fold[1]] = { [fold[2]]: fold };
+  }
+}
+
+const scKeys = Object.keys(scData).sort(sortHex);
+const cfKeys = Object.keys(cfData).sort(sortHex);
+const keys = scKeys.concat(cfKeys).sort(sortHex);
+
+const data = {};
+
+const hex = a => a ? '0x'+a : '0';
+const hexes = a => a ? '[' + a.split(' ').filter(b=>b).map(hex) + ']' : '[]';
+
+for (let code of keys) {
+  const d = { code: hex(code) };
+  const sc = scData[code];
+  d.lower = hexes(sc?.[2]);
+  d.title = hexes(sc?.[3]);
+  d.upper = hexes(sc?.[4]);
+  const cf = cfData[code];
+  d.fold = hex(cf?.S?.[3] || cf?.C?.[3]);
+  d.foldFull = hexes(cf?.F?.[3] || cf?.C?.[3]);
+  data[code] = d;
+}
+
+const lines = keys.map(code => data[code]).map(d =>
+  `{ code: ${d.code}, lower: ${d.lower}, title: ${d.title}, upper: ${d.upper}, fold: ${d.fold}, foldFull: ${d.foldFull} }`
+);
+
+
+
+const file = `-----------------------------------------------------------
+-- This is an automatically generated file: do not edit"
+-- Generated by fullcase.js on ${new Date().toDateString()}
+-----------------------------------------------------------
+
+module Data.CodePoint.Unicode.Internal.Casing where
+
+import Prelude
+
+import Data.Array as Array
+import Data.CodePoint.Unicode.Internal (bsearch, uTowlower, uTowtitle, uTowupper)
+import Data.Maybe (Maybe(..))
+
+type CaseRec =
+  {
+    code :: Int,
+    lower :: Array Int,
+    title :: Array Int,
+    upper :: Array Int,
+    fold :: Int,
+    foldFull :: Array Int
+  }
+
+rules :: Array CaseRec
+rules = [
+  ${lines.join(",\n  ")}
+]
+
+zeroRec :: Int -> CaseRec
+zeroRec code = { code, lower: [], title: [], upper: [], fold: 0, foldFull: [] }
+
+recCmp :: CaseRec -> CaseRec -> Ordering
+recCmp { code } { code: code' } = compare code code'
+
+findRule :: Int -> CaseRec
+findRule code = case bsearch (zeroRec code) rules (Array.length rules) recCmp of
+  Nothing -> zeroRec code
+  Just r -> r
+
+fold :: Int -> Int
+fold code =
+  let folded = (findRule code).fold
+  in if folded == 0 then uTowlower code else folded
+
+foldFull :: Int -> Array Int
+foldFull code =
+  let folded = (findRule code).foldFull
+  in if Array.null folded then [uTowlower code] else folded
+
+lower :: Int -> Array Int
+lower code =
+  let lowered = (findRule code).lower
+  in if Array.null lowered then [uTowlower code] else lowered
+
+title :: Int -> Array Int
+title code =
+  let titled = (findRule code).title
+  in if Array.null titled then [uTowtitle code] else titled
+
+upper :: Int -> Array Int
+upper code =
+  let uppered = (findRule code).upper
+  in if Array.null uppered then [uTowupper code] else uppered
+`;
+
+fs.writeFileSync("src/Data/CodePoint/Unicode/Internal/Casing.purs", file);