From 77db1562a47acfebd27affecb8d63f3830778c0d Mon Sep 17 00:00:00 2001 From: Verity Scheel Date: Wed, 24 Nov 2021 21:46:12 -0500 Subject: [PATCH 1/2] Update scripts to take into account purs-tidy formatting --- docs/README.md | 1 + fullcase.js | 51 +-- src/Data/CodePoint/Unicode/Internal.purs | 2 +- .../CodePoint/Unicode/Internal/Casing.purs | 2 +- ubconfc | 323 ++++++++++-------- 5 files changed, 219 insertions(+), 160 deletions(-) diff --git a/docs/README.md b/docs/README.md index f2bc1ef..17c41f3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ The [Data.CodePoint.Unicode.Internal](../src/Data/CodePoint/Unicode/Internal.pur $ ./download.sh $ ./ubconfc < UnicodeData.txt > src/Data/CodePoint/Unicode/Internal.purs $ ./fullcase.js +$ purs-tidy format-in-place src/Data/CodePoint/Unicode/Internal.purs src/Data/CodePoint/Unicode/Internal/Casing.purs ``` (Note that this downloads data according to the version stored in the [`unicode-version`](../unicode-version) file in the root.) diff --git a/fullcase.js b/fullcase.js index 94bbb6d..2b7ab80 100755 --- a/fullcase.js +++ b/fullcase.js @@ -36,7 +36,7 @@ const keys = scKeys.concat(cfKeys).sort(sortHex); const data = {}; const hex = a => a ? '0x'+a : '0'; -const hexes = a => a ? '[' + a.split(' ').filter(b=>b).map(hex) + ']' : '[]'; +const hexes = a => a ? '[ ' + a.split(' ').filter(b=>b).map(hex).join(', ') + ' ]' : '[]'; for (let code of keys) { const d = { code: hex(code) }; @@ -70,19 +70,18 @@ import Data.CodePoint.Unicode.Internal (bsearch, uTowlower, uTowtitle, uTowupper import Data.Maybe (Maybe(..)) type CaseRec = - { - code :: Int, - lower :: Array Int, - title :: Array Int, - upper :: Array Int, - fold :: Int, - foldFull :: Array Int + { code :: Int + , lower :: Array Int + , title :: Array Int + , upper :: Array Int + , fold :: Int + , foldFull :: Array Int } rules :: Array CaseRec -rules = [ - ${lines.join(",\n ")} -] +rules = + ${lines.map((v, i) => i ? ", " + v : "[ " + v).join("\n ")} + ] zeroRec :: Int -> CaseRec zeroRec code = { code, lower: [], title: [], upper: [], fold: 0, foldFull: [] } @@ -97,28 +96,38 @@ findRule code = case bsearch (zeroRec code) rules (Array.length rules) recCmp of fold :: Int -> Int fold code = - let folded = (findRule code).fold - in if folded == 0 then uTowlower code else folded + let + folded = (findRule code).fold + in + if folded == 0 then uTowlower code else folded foldFull :: Int -> Array Int foldFull code = - let folded = (findRule code).foldFull - in if Array.null folded then [uTowlower code] else folded + let + folded = (findRule code).foldFull + in + if Array.null folded then [ uTowlower code ] else folded lower :: Int -> Array Int lower code = - let lowered = (findRule code).lower - in if Array.null lowered then [uTowlower code] else lowered + let + lowered = (findRule code).lower + in + if Array.null lowered then [ uTowlower code ] else lowered title :: Int -> Array Int title code = - let titled = (findRule code).title - in if Array.null titled then [uTowtitle code] else titled + let + titled = (findRule code).title + in + if Array.null titled then [ uTowtitle code ] else titled upper :: Int -> Array Int upper code = - let uppered = (findRule code).upper - in if Array.null uppered then [uTowupper code] else uppered + let + uppered = (findRule code).upper + in + if Array.null uppered then [ uTowupper code ] else uppered `; fs.writeFileSync("src/Data/CodePoint/Unicode/Internal/Casing.purs", file); diff --git a/src/Data/CodePoint/Unicode/Internal.purs b/src/Data/CodePoint/Unicode/Internal.purs index a02ac51..d63f7b2 100644 --- a/src/Data/CodePoint/Unicode/Internal.purs +++ b/src/Data/CodePoint/Unicode/Internal.purs @@ -1,6 +1,6 @@ ----------------------------------------------------------- -- This is an automatically generated file: do not edit --- Generated by ubconfc at Tue Jan 12 18:57:20 CST 2021 +-- Generated by ubconfc at Wed Nov 24 21:40:34 EST 2021 ----------------------------------------------------------- module Data.CodePoint.Unicode.Internal where diff --git a/src/Data/CodePoint/Unicode/Internal/Casing.purs b/src/Data/CodePoint/Unicode/Internal/Casing.purs index 02ddf44..83cbb00 100644 --- a/src/Data/CodePoint/Unicode/Internal/Casing.purs +++ b/src/Data/CodePoint/Unicode/Internal/Casing.purs @@ -1,6 +1,6 @@ ----------------------------------------------------------- -- This is an automatically generated file: do not edit" --- Generated by fullcase.js on Tue Jan 26 2021 +-- Generated by fullcase.js on Wed Nov 24 2021 ----------------------------------------------------------- module Data.CodePoint.Unicode.Internal.Casing where diff --git a/ubconfc b/ubconfc index 1f292af..bb18f07 100755 --- a/ubconfc +++ b/ubconfc @@ -27,7 +27,6 @@ echo # Define structures cat < show rule.category - <> ", unicodeCat: " <> show rule.unicodeCat - <> ", possible: " <> show rule.possible - <> ", updist: " <> show rule.updist - <> ", lowdist: " <> show rule.lowdist - <> ", titledist: " <> show rule.titledist - <> " }" - -newtype CharBlock = CharBlock { start :: Int - , length :: Int - , convRule :: ConversionRule - } + show (ConversionRule rule) = "ConversionRule { category: " <> show rule.category + <> ", unicodeCat: " + <> show rule.unicodeCat + <> ", possible: " + <> show rule.possible + <> ", updist: " + <> show rule.updist + <> ", lowdist: " + <> show rule.lowdist + <> ", titledist: " + <> show rule.titledist + <> " }" + +newtype CharBlock = CharBlock + { start :: Int + , length :: Int + , convRule :: ConversionRule + } instance showCharBlock :: Show CharBlock where - show (CharBlock rule) = "CharBlock { start: " <> show rule.start - <> ", length: " <> show rule.length - <> ", convRule: " <> show rule.convRule - <> " }" - + show (CharBlock rule) = "CharBlock { start: " <> show rule.start + <> ", length: " + <> show rule.length + <> ", convRule: " + <> show rule.convRule + <> " }" EOF @@ -285,31 +293,31 @@ END { } print "allchars :: Array CharBlock" - printf "allchars = [ " + print "allchars =" for(i = 0; i < blockidx; i++) { - printf (i>0) ? " , " : "" + printf (i>0) ? " , " : " [ " printf blocks[i] "\n" } - print " ]\n" + print " ]\n" print "convchars :: Array CharBlock" - printf "convchars = [ " + print "convchars =" for(i = 0; i < cblckidx; i++) { - printf (i>0) ? " , " : "" + printf (i>0) ? " , " : " [ " printf cblcks[i] "\n" } - print " ]\n" + print " ]\n" print "spacechars :: Array CharBlock" - printf "spacechars = [ " + print "spacechars =" for(i = 0; i < sblckidx; i++) { - printf (i>0) ? " , " : "" + printf (i>0) ? " , " : " [ " printf sblcks[i] "\n" } - print " ]\n" + print " ]" } ' # Output the C procedures code @@ -331,34 +339,39 @@ nullrule = ConversionRule { category: gencatLU, unicodeCat: NUMCAT_CN, possible: blkCmp :: CharBlock -> CharBlock -> Ordering blkCmp (CharBlock key) (CharBlock cur) - | key.start >= cur.start && key.start < (cur.start + cur.length) = EQ - | key.start > cur.start = GT - | otherwise = LT - + | key.start >= cur.start && key.start < (cur.start + cur.length) = EQ + | key.start > cur.start = GT + | otherwise = LT -- TODO: This might be better if it returned Nothing instead of the nullrule. getRule :: Array CharBlock -> Int -> Int -> Maybe ConversionRule getRule blocks unichar size = - -- TODO: does using the nullrule make sense here? - let key = CharBlock { start: unichar, length: 1, convRule: nullrule } - maybeCharBlock = bsearch key blocks size blkCmp - in case maybeCharBlock of - Nothing -> Nothing - Just (CharBlock charBlock) -> Just charBlock.convRule + -- TODO: does using the nullrule make sense here? + let + key = CharBlock { start: unichar, length: 1, convRule: nullrule } + maybeCharBlock = bsearch key blocks size blkCmp + in + case maybeCharBlock of + Nothing -> Nothing + Just (CharBlock charBlock) -> Just charBlock.convRule -- The third argument is a size. It says to only look at this many elements -- in the array. There are times when we don't want to look at ALL the -- elements, but only a subset. -bsearch :: forall a . a -> Array a -> Int -> (a -> a -> Ordering) -> Maybe a +bsearch :: forall a. a -> Array a -> Int -> (a -> a -> Ordering) -> Maybe a bsearch a array size compare = go 0 size - where - go i k | i > k || i >= Array.length array = Nothing - | otherwise = let j = floor (toNumber (i + k) / 2.0) - b = unsafePartial (Array.unsafeIndex array j) - in case compare a b of - EQ -> Just b - GT -> go (j + 1) k - _ -> go i (j - 1) + where + go i k + | i > k || i >= Array.length array = Nothing + | otherwise = + let + j = floor (toNumber (i + k) / 2.0) + b = unsafePartial (Array.unsafeIndex array j) + in + case compare a b of + EQ -> Just b + GT -> go (j + 1) k + _ -> go i (j - 1) -- Check whether a character (internal code) has certain attributes. -- Attributes (category flags) may be ORed. The function ANDs @@ -367,29 +380,52 @@ bsearch a array size compare = go 0 size -- the result will be nonzero. checkAttr :: Array Int -> Int -> Boolean checkAttr categories char = - let numOfBlocks = if char < 256 then numLat1Blocks else numBlocks - maybeConversionRule = getRule allchars char numOfBlocks - in case maybeConversionRule of - Nothing -> false - Just (ConversionRule rule) -> isJust (Array.elemIndex rule.category categories) + let + numOfBlocks = if char < 256 then numLat1Blocks else numBlocks + maybeConversionRule = getRule allchars char numOfBlocks + in + case maybeConversionRule of + Nothing -> false + Just (ConversionRule rule) -> isJust (Array.elemIndex rule.category categories) checkAttrS :: Array Int -> Int -> Boolean checkAttrS categories char = - let maybeConversionRule = getRule spacechars char numSpaceBlocks - in case maybeConversionRule of - Nothing -> false - Just (ConversionRule rule) -> isJust (Array.elemIndex rule.category categories) + let + maybeConversionRule = getRule spacechars char numSpaceBlocks + in + case maybeConversionRule of + Nothing -> false + Just (ConversionRule rule) -> isJust (Array.elemIndex rule.category categories) uIswcntrl :: Int -> Boolean uIswcntrl = checkAttr [ gencatCC ] uIswprint :: Int -> Boolean uIswprint = checkAttr - [ gencatMC, gencatNO, gencatSK, gencatME, gencatND, gencatPO - , gencatLT, gencatPC, gencatSM, gencatZS, gencatLU, gencatPD - , gencatSO, gencatPE, gencatPF, gencatPS, gencatSC, gencatLL - , gencatLM, gencatPI, gencatNL, gencatMN, gencatLO - ] + [ gencatMC + , gencatNO + , gencatSK + , gencatME + , gencatND + , gencatPO + , gencatLT + , gencatPC + , gencatSM + , gencatZS + , gencatLU + , gencatPD + , gencatSO + , gencatPE + , gencatPF + , gencatPS + , gencatSC + , gencatLL + , gencatLM + , gencatPI + , gencatNL + , gencatMN + , gencatLO + ] uIswupper :: Int -> Boolean uIswupper = checkAttr [ gencatLU, gencatLT ] @@ -405,9 +441,18 @@ uIswdigit = checkAttr [ gencatND ] uIswalnum :: Int -> Boolean uIswalnum = checkAttr - [ gencatLT, gencatLU, gencatLL, gencatLM, gencatLO, gencatMC, gencatME - , gencatMN, gencatNO, gencatND, gencatNL - ] + [ gencatLT + , gencatLU + , gencatLL + , gencatLM + , gencatLO + , gencatMC + , gencatME + , gencatMN + , gencatNO + , gencatND + , gencatNL + ] uIswspace :: Int -> Boolean uIswspace = checkAttrS [ gencatZS ] @@ -418,10 +463,12 @@ uIswspace = checkAttrS [ gencatZS ] caseConv :: (ConversionRule -> Int) -> Int -> Int caseConv f char = - let maybeConversionRule = getRule convchars char numConvBlocks - in case maybeConversionRule of - Nothing -> char - Just conversionRule -> char + f conversionRule + let + maybeConversionRule = getRule convchars char numConvBlocks + in + case maybeConversionRule of + Nothing -> char + Just conversionRule -> char + f conversionRule uTowupper :: Int -> Int uTowupper = caseConv (\(ConversionRule rule) -> rule.updist) @@ -434,7 +481,9 @@ uTowtitle = caseConv (\(ConversionRule rule) -> rule.titledist) uGencat :: Int -> Maybe UnicodeCategory uGencat char = - let conversionRule = getRule allchars char numBlocks - in map (\(ConversionRule rule) -> rule.unicodeCat) conversionRule + let + conversionRule = getRule allchars char numBlocks + in + map (\(ConversionRule rule) -> rule.unicodeCat) conversionRule EOF From a4ab51f4ceb5aefdaa6a3a4fbe09770b6946aad1 Mon Sep 17 00:00:00 2001 From: Verity Scheel Date: Wed, 24 Nov 2021 21:46:55 -0500 Subject: [PATCH 2/2] Merge docs/README.md into CONTRIBUTING.md --- CONTRIBUTING.md | 13 +++++++++++++ docs/README.md | 16 ---------------- 2 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 docs/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e98e9ad..2b80c04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,3 +3,16 @@ Thanks for your interest in contributing to `unicode`! We welcome new contributions regardless of your level of experience or familiarity with PureScript. Every library in the Contributors organization shares a simple handbook that helps new contributors get started. With that in mind, please [read the short contributing guide on purescript-contrib/governance](https://github.com/purescript-contrib/governance/blob/main/contributing.md) before contributing to this library. + +## Generating Internal Modules + +The [Data.CodePoint.Unicode.Internal](../src/Data/CodePoint/Unicode/Internal.purs) and [Data.CodePoint.Unicode.Internal.Casing](../src/Data/CodePoint/Unicode/Internal/Casing.purs) modules can be generated with the following command from the root of this repository: + +```sh +$ ./download.sh +$ ./ubconfc < UnicodeData.txt > src/Data/CodePoint/Unicode/Internal.purs +$ ./fullcase.js +$ purs-tidy format-in-place src/Data/CodePoint/Unicode/Internal.purs src/Data/CodePoint/Unicode/Internal/Casing.purs +``` + +(Note that this downloads data according to the version stored in the [`unicode-version`](../unicode-version) file in the root.) diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 17c41f3..0000000 --- a/docs/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Unicode Documentation - -This directory contains documentation for `unicode`. If you are interested in contributing new documentation, please read the [contributor guidelines](../CONTRIBUTING.md) and [What Nobody Tells You About Documentation](https://documentation.divio.com) for help getting started. - -## Generating Internal Modules - -The [Data.CodePoint.Unicode.Internal](../src/Data/CodePoint/Unicode/Internal.purs) and [Data.CodePoint.Unicode.Internal.Casing](../src/Data/CodePoint/Unicode/Internal/Casing.purs) modules can be generated with the following command from the root of this repository: - -```sh -$ ./download.sh -$ ./ubconfc < UnicodeData.txt > src/Data/CodePoint/Unicode/Internal.purs -$ ./fullcase.js -$ purs-tidy format-in-place src/Data/CodePoint/Unicode/Internal.purs src/Data/CodePoint/Unicode/Internal/Casing.purs -``` - -(Note that this downloads data according to the version stored in the [`unicode-version`](../unicode-version) file in the root.)