Hi,
to summarize, it is an ETL,
just took your case as an example for my experiments here
https://github.com/mh-cbon/poc-pipe
// demo csv read/write.
func csvdemo() {
// f, err := os.Open("C:/ORIG.csv")
// if err != nil {
// panic(err)
// }
var b bytes.Buffer
b.Write([]byte(csvdata))
src := t.NewCsvReader(&b)
src.
Pipe(&processCsvRecords{}).
Pipe(&t.CsvWriter{}).
Pipe(t.NewBytesPrefixer("", "\n")).
Sink(t.NewByteSink(os.Stdout))
if err := src.Consume(); err != nil {
panic(err)
}
}
// Calculate economic block value given a 40x20x12 block of density sg and grade
func calculateE(sg float64, grade float64) float64 {
return 40 * 20 * 12 * sg * grade * 80 / 10 * 0.66
}
type processCsvRecords struct {
t.StringSliceStream
d bool
}
func (p *processCsvRecords) Write(rec []string) error {
// rewrite headers
if !p.d {
p.d = true
return p.StringSliceStream.Write(
[]string{"set", "headers", "as", "you", "d", "like",
"it"},
)
}
if len(rec) < 17 {
return fmt.Errorf("Line too short %#v", rec)
}
sg, _ := strconv.ParseFloat(rec[17], 64)
grade, _ := strconv.ParseFloat(rec[13], 64)
if grade < 0 {
grade = 0
} else {
grade = 0.1 * grade
}
if sg < 0 {
sg = 0
}
// fmt.Println(grade)
// get economic block value
e := calculateE(sg, grade)
gradeStr := strconv.FormatFloat(grade, 'f', 8, 64)
sgStr := strconv.FormatFloat(sg, 'f', 8, 64)
eStr := strconv.FormatFloat(e, 'f', 8, 64)
return p.StringSliceStream.Write(
append(rec[0:4], eStr, sgStr, gradeStr),
)
}
// 500mb csv with head of the data looks like:
const csvdata =
`xcentre,ycentre,zcentre,xlength,ylength,zlength,fe_percent,fe_recovery,oxide,rescat,sg,mat_type_8,fillpc,mass_recovery,mass_recovery_percent,air,al2o3,cao,k2o,loi,mgo,mno,phos,sio2,tio2
556960.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
557000.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
556960.000,6320000.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
557000.000,6320000.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
556960.000,6319980.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
557000.000,6319980.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
556960.000,6320000.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
557000.000,6320000.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
557040.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
`
On Friday, February 24, 2017 at 2:58:52 PM UTC+1, Robbie Wright wrote:
>
> Hi this is my first go program and I am looking for some feedback and help
> writing buffer to csv. To give you an idea of the structure of the data I
> have run head on the input file gives output seen at the bottom.
>
> package main
>
> import (
> "bytes"
> "encoding/csv"
> "fmt"
> "io"
> "log"
> "strconv"
> "strings"
> "bufio"
> "io/ioutil"
> )
>
> // Calculate economic block value given a 40x20x12 block of density sg and
> grade
> func calculateE(sg float64, grade float64) float64 {
> return 40 * 20 * 12 * sg * grade * 80 / 10 * 0.66
> }
>
> // Credit: Stack Overflow
> func processCSV(rc io.Reader) (ch chan []string) {
> ch = make(chan []string, 10)
> go func() {
> r := csv.NewReader(rc)
> if _, err := r.Read(); err != nil { //read header
> log.Fatal(err)
> }
> defer close(ch)
> for {
> rec, err := r.Read()
>
> if err != nil {
> if err == io.EOF {
> break
> }
> log.Fatal(err)
>
> }
> ch <- rec
> }
> }()
> return
> }
>
> // Credit: Stack Overflow
> func Readln(r *bufio.Reader) (string, error) {
> var (
> isPrefix bool = true
> err error = nil
> line, ln []byte
> )
> for isPrefix && err == nil {
> line, isPrefix, err = r.ReadLine()
> ln = append(ln, line...)
> }
> return string(ln), err
> }
>
> func main() {
> var buf bytes.Buffer
>
> w := csv.NewWriter(&buf)
> file := "C:/ORIG.csv"
> dat, _ := ioutil.ReadFile(file)
> f := string(dat)
>
> for rec := range processCSV(strings.NewReader(f)) {
> sg, err := strconv.ParseFloat(rec[17], 64)
> grade, err := strconv.ParseFloat(rec[13], 64)
>
> if grade < 0 {
> grade = 0
> } else {
> grade = 0.1*grade
> }
> if sg < 0 {
> sg = 0
> }
> fmt.Println(grade)
> // get economic block value
> e := calculateE(sg, grade)
> gradeStr := strconv.FormatFloat(grade, 'f', 8, 64)
> sgStr := strconv.FormatFloat(sg, 'f', 8, 64)
> eStr := strconv.FormatFloat(e, 'f', 8, 64)
> rec := append(rec[0:4], eStr, sgStr, gradeStr)
> if err = w.Write(rec); err != nil {
> log.Fatal(err)
> }
> }
> w.Flush()
> if err := w.Error(); err != nil {
> log.Fatal(err)
> }
> //fmt.Println(buf.String())
> //f2, err := os.Create("C:/Users/root/result.csv")
> //defer f2.Close()
> //writer := csv.NewWriter(f2)
> //writer.Write(buf.String())
> }
>
> // 500mb csv with head of the data looks like:
> const data =
> `xcentre,ycentre,zcentre,xlength,ylength,zlength,fe_percent,fe_recovery,oxide,rescat,sg,mat_type_8,fillpc,mass_recovery,mass_recovery_percent,air,al2o3,cao,k2o,loi,mgo,mno,phos,sio2,tio2
> 556960.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 557000.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 556960.000,6320000.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 557000.000,6320000.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 556960.000,6319980.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 557000.000,6319980.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 556960.000,6320000.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 557000.000,6320000.000,-1088.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,100,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> 557040.000,6319980.000,-1100.000,40.000,20.000,12.000,-99.00000000,66.00000000,-99,4,2.84999990,2,91,0.00000000,0.00000000,0,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000
> `
>
>
>
>
--
You received this message because you are subscribed to the Google Groups
"golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
