change(prime): Calculated up to 400,000,000

go.mod

@@ -2,6 +2,15 @@ module datalore/prime-div
 
 go 1.23.0
 
-require github.com/wamuir/graft v0.9.0
+require (
+	github.com/klauspost/compress v1.17.10
+	github.com/vmihailenco/msgpack v4.0.4+incompatible
+	github.com/wamuir/graft v0.9.0
+)
 
-require google.golang.org/protobuf v1.34.2 // indirect
+require (
+	github.com/golang/protobuf v1.5.2 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+)

go.sum

@@ -1,8 +1,50 @@
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
+github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
+github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/wamuir/graft v0.9.0 h1:5DbPtr3MfWRq9bFHivbbvNic8h8jtcKK12Rxk0644iY=
 github.com/wamuir/graft v0.9.0/go.mod h1:k6NJX3fCM/xzh5NtHky9USdgHTcz2vAvHp4c23I6UK4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

io.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/vmihailenco/msgpack"
+	"github.com/klauspost/compress/zstd"
+)
+
+func loadPrimes(useTxt bool) []int64 {
+	if useTxt {
+		return loadPrimesTxt()
+	} else {
+		return loadPrimesMsgpack()
+	}
+}
+
+func loadPrimesTxt() []int64 {
+	file := must(os.Open("prime.txt"))
+	defer file.Close()
+
+	var primes []int64
+
+	scanner := bufio.NewScanner(file)
+
+	for scanner.Scan() {
+		nextPrime := must(strconv.ParseInt(scanner.Text(), 10, 64))
+		primes = append(primes, nextPrime)
+	}
+
+	return primes
+}
+
+func loadPrimesMsgpack() []int64 {
+	file := must(os.Open("prime.msgpack.zst"))
+	defer file.Close()
+
+	zstdDecoder := must(zstd.NewReader(file))
+	defer zstdDecoder.Close()
+
+	var primes []int64
+	msgpackDecoder := msgpack.NewDecoder(zstdDecoder)
+	err := msgpackDecoder.Decode(&primes)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return primes
+}
+
+func writePrimes(primes []int64, useTxt bool) {
+	startTime := time.Now()
+
+	if useTxt {
+		writePrimesTxt(primes)
+	} else {
+		writePrimesMsgpack(primes)
+	}
+
+	fmt.Printf("File writing took %v\n", time.Now().Sub(startTime))
+}
+
+func writePrimesTxt(primes []int64) {
+	file := must(os.Create("prime.txt"))
+	defer file.Close()
+
+	writer := bufio.NewWriter(file)
+
+	for _, prime := range primes {
+		writer.WriteString(fmt.Sprintf("%d\n", prime))
+	}
+
+	writer.Flush()
+}
+
+func writePrimesMsgpack(primes []int64) {
+	file := must(os.Create("prime.msgpack.zst"))
+	defer file.Close()
+
+	zstdEncoder := must(zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression)))
+	defer zstdEncoder.Close()
+	
+	msgpackEncoder := msgpack.NewEncoder(zstdEncoder)
+	err := msgpackEncoder.Encode(&primes)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

main.go

@@ -1,19 +1,27 @@
 package main
 
 import (
-	"bufio"
 	"context"
 	"flag"
 	"fmt"
 	"log"
-	"os"
 	"runtime"
 	"strconv"
 	"time"
+	"os"
 )
 
-func calculateProgress(start, end, step, index int64) float32 {
-	steps := (end - start) / step
+type Config struct {
+	PrimeList bool
+	DontLoad bool
+	NumRoutines int
+	UseTensors bool
+	UseTxtRead bool
+	UseTxtWrite bool
+}
+
+func calculateProgress(start, end, index int64) float32 {
+	steps := (end - start)
 	current := index - start
 	if current == 0 {
 		return 0
@@ -56,15 +64,15 @@ func checkIsDivisibleByPrime(number int64, offset, stride int, primes *[]int64,
 	resultChannel <- false
 }
 
-func generatePrimes(upperLimit int64, loadList bool, numRoutines int, useTensors bool) []int64 {
+func generatePrimes(upperLimit int64, config Config) []int64 {
 	var primes []int64
 
 	bootTime := time.Now()
 
-	if loadList {
-		primes = loadPrimes()
-	} else {
+	if config.DontLoad {
 		primes = []int64{2, 3, 5}
+	} else {
+		primes = loadPrimes(config.UseTxtRead)
 	}
 
 	fmt.Printf("Startup time: %v\n", time.Now().Sub(bootTime))
@@ -74,10 +82,10 @@ func generatePrimes(upperLimit int64, loadList bool, numRoutines int, useTensors
 
 	startTime := time.Now()
 
-	if useTensors {
+	if config.UseTensors {
 		primes = generatePrimesGPU(upperLimit, primes, progress)
 	} else {
-		primes = generatePrimesCPU(upperLimit, primes, numRoutines, progress)
+		primes = generatePrimesCPU(upperLimit, primes, config.NumRoutines, progress)
 	}
 	close(progress)
 
@@ -95,9 +103,10 @@ func generatePrimesCPU(upperLimit int64, primes []int64, numRoutines int, progre
 
 	fmt.Printf("Calculating with %d routines\n\n", numRoutines)
 
-	for i := (primes[len(primes) - 1] + 2); i <= upperLimit; i += 2 {
+	continueFrom := primes[len(primes) - 1] + 2
+	for i := continueFrom; i <= upperLimit; i += 2 {
 		select {
-		case progress <- calculateProgress(6, upperLimit, 1, i):
+		case progress <- calculateProgress(continueFrom, upperLimit, i):
 		default:
 		}
 
@@ -139,7 +148,7 @@ func calculatePrimeParts(number int64, primes []int64) []int64 {
 
 	for i := int64(len(primes) - 1); i >= 0; i-- {
 		select {
-		case progress <- 1.0 - calculateProgress(0, int64(len(primes) - 1), 1, i):
+		case progress <- 1.0 - calculateProgress(0, int64(len(primes) - 1), i):
 		default:
 		}
 		flooredDiv := number / primes[i]
@@ -160,17 +169,14 @@ func calculatePrimeParts(number int64, primes []int64) []int64 {
 }
 
 func main() {
-	var (
-		primeList bool
-		dontLoad bool
-		numRoutines int
-		useTensors bool
-	)
+	var config Config
 
-	flag.BoolVar(&primeList, "p", false, "Only calculate and print prime list")
-	flag.BoolVar(&dontLoad, "d", false, "Don't load precalculated primes, calculate from 0")
-	flag.IntVar(&numRoutines, "r", 0, "How many routines to use for calculation. 0 = number of available CPU cores")
-	flag.BoolVar(&useTensors, "t", false, "Use tensorflow")
+	flag.BoolVar(&config.PrimeList, "p", false, "Only calculate and print prime list")
+	flag.BoolVar(&config.DontLoad, "d", false, "Don't load precalculated primes, calculate from 0")
+	flag.IntVar(&config.NumRoutines, "r", 0, "How many routines to use for calculation. 0 = number of available CPU cores")
+	flag.BoolVar(&config.UseTensors, "t", false, "Use tensorflow")
+	flag.BoolVar(&config.UseTxtRead, "read-legacy", false, "Read legacy prime.txt file instead of prime.msgpack")
+	flag.BoolVar(&config.UseTxtWrite, "write-legacy", false, "Write legacy prime.txt file instead of prime.msgpack")
 	flag.Parse()
 	numStr := flag.Arg(0)
 
@@ -185,33 +191,20 @@ func main() {
 		log.Fatal(err)
 	}
 
-	if primeList {
-		onlyGenerate(number, dontLoad, numRoutines, useTensors)
+	if config.PrimeList {
+		onlyGenerate(number, config)
 	} else {
-		calculate(number, dontLoad, numRoutines, useTensors)
+		calculate(number, config)
 	}
 }
 
-func onlyGenerate(number int64, dontLoad bool, numRoutines int, useTensors bool) {
-	primes := generatePrimes(number, !dontLoad, numRoutines, useTensors)
-
-	file, err := os.Create("prime.txt")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	writer := bufio.NewWriter(file)
-
-	for _, prime := range primes {
-		writer.WriteString(fmt.Sprintf("%d\n", prime))
-	}
-
-	writer.Flush()
-	file.Close()
+func onlyGenerate(number int64, config Config) {
+	primes := generatePrimes(number, config)
+	writePrimes(primes, config.UseTxtWrite)
 }
 
-func calculate(number int64, dontLoad bool, numRoutines int, useTensors bool) {
-	primes := generatePrimes(number, !dontLoad, numRoutines, useTensors)
+func calculate(number int64, config Config) {
+	primes := generatePrimes(number, config)
 
 	primeParts := calculatePrimeParts(number, primes)
 
@@ -229,26 +222,3 @@ func calculate(number int64, dontLoad bool, numRoutines int, useTensors bool) {
 	print(" = ")
 	println(sum)
 }
-
-func loadPrimes() []int64 {
-	file, err := os.Open("prime.txt")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	var primes []int64
-
-	scanner := bufio.NewScanner(file)
-
-	for scanner.Scan() {
-		nextPrime, err := strconv.ParseInt(scanner.Text(), 10, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		primes = append(primes, nextPrime)
-	}
-
-	file.Close()
-
-	return primes
-}

main_test.go

@@ -0,0 +1,21 @@
+package main
+
+import "testing"
+
+func testGeneratePrimes(upperLimit int64, useTensors bool, b *testing.B) {
+	var config Config
+	config.DontLoad = true
+	config.UseTensors = useTensors
+
+	for i := 0; i < b.N; i++ {
+		generatePrimes(upperLimit, config)
+	}
+}
+
+func BenchmarkPrimes1000CPU(b *testing.B) {
+	testGeneratePrimes(1000, false, b)
+}
+
+func BenchmarkPrimes1000GPU(b *testing.B) {
+	testGeneratePrimes(1000, true, b)
+}

tensor.go

@@ -26,9 +26,10 @@ func generatePrimesGPU(upperLimit int64, primes []int64, progress chan float32)
 
 	fmt.Printf("Calculating with tensorflow\n\n")
 
-	for i := (primes[len(primes) - 1] + 2); i <= upperLimit; i += 2 {
+	continueFrom := primes[len(primes) - 1] + 2
+	for i := continueFrom; i <= upperLimit; i += 2 {
 		select {
-		case progress <- calculateProgress(6, upperLimit, 1, i):
+		case progress <- calculateProgress(continueFrom, upperLimit, i):
 		default:
 		}