From 797f9d11e7be7e406fd6b29399fc5e90cf389c88 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 7 May 2015 14:52:59 +0300 Subject: [PATCH] implement simple in memory caching closes #13 Signed-off-by: Vasiliy Tolstov --- main.go | 83 +++++++++++++++++++++++++------ server.go | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 210 insertions(+), 17 deletions(-) diff --git a/main.go b/main.go index d48d958..3ca3c61 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,14 @@ func handle(c net.Conn) { if err != nil { fmt.Printf(err.Error()) } + /* + s.cache = groupcache.NewGroup("cache", 10 * 1024, groupcache.GetterFunc( + func(ctx groupcache.Context, key string, dest groupcache.Sink) error { + contents, err := ioutil.ReadFile(key) + dest.SetBytes(contents) + return err + })) + */ s.ctrl.PrintfLine("220 %s [%s]", "go-ftp", c.LocalAddr()) for { line, err := s.ctrl.ReadLine() @@ -59,6 +67,8 @@ func handle(c net.Conn) { s.cmdServerCdup(params) case "RNFR": s.cmdServerRnfr(params) + case "RNTO": + s.cmdServerRnto(params) case "OPTS": s.cmdServerOpts(params) case "MDTM": @@ -87,6 +97,8 @@ func handle(c net.Conn) { s.cmdServerQuit(params) s.Close() return + case "HELP": + s.cmdServerHelp(params) case "SIZE": s.cmdServerSize(params) case "CWD", "XCWD": @@ -191,20 +203,33 @@ func (s *Conn) cmdServerList(args string) { } else { if s.path != "/" { cnt = strings.Split(s.path, "/")[1] - p = args + if len(args) > 0 { + p = args + } else { + p = filepath.Clean(strings.Join(strings.Split(s.path, "/")[2:], "/")) + } fmt.Printf("cnt1 path %s cnt %s p %s\n", s.path, cnt, p) } else { fmt.Printf("cnt2 %s\n", s.path) - cnt = strings.Split(args, "/")[0] - p = strings.Join(strings.Split(args, "/")[1:], "/") + if len(args) > 0 { + cnt = strings.Split(args, "/")[0] + p = strings.Join(strings.Split(args, "/")[1:], "/") + } else { + cnt = strings.Split(s.path, "/")[1] + p = filepath.Clean(strings.Join(strings.Split(s.path, "/")[2:], "/")) + } } } fmt.Printf("cnt3 %s p %s\n", cnt, p) if cnt == "" { - cnts, err := s.sw.ContainersAll(nil) + cnts, err := s.loadContainers() if err != nil { - fmt.Printf(err.Error()) - return + cnts, err = s.sw.ContainersAll(nil) + if err != nil { + fmt.Printf(err.Error()) + return + } + s.saveContainers(cnts) } files = append(files, NewDirItem(".", 4096, 0), NewDirItem("..", 4096, 0)) for _, ct := range cnts { @@ -218,18 +243,28 @@ func (s *Conn) cmdServerList(args string) { opts := &swift.ObjectsOpts{Delimiter: '/'} fmt.Printf("cnt4: %s p: %s\n", cnt, p) if p != "." { - opts.Path = p + opts.Prefix = p + if !strings.HasSuffix(opts.Prefix, "/") { + opts.Prefix += "/" + } } - objs, err := s.sw.ObjectsAll(cnt, opts) + objs, err := s.loadObjects(cnt, opts) if err != nil { - fmt.Printf(err.Error()) - return + objs, err = s.sw.ObjectsAll(cnt, opts) + if err != nil { + fmt.Printf(err.Error()) + return + } + s.saveObjects(cnt, objs, opts) } fmt.Printf("%+v\n", objs) files = append(files, NewDirItem(".", 4096, 0), NewDirItem("..", 4096, 0)) var it os.FileInfo for _, obj := range objs { + if obj.SubDir != "" || strings.Index(obj.Name, "/") > 0 { + continue + } if obj.PseudoDirectory || obj.ContentType == "application/directory" { it = NewDirItem(obj.Name, obj.Bytes, 1) } else { @@ -274,10 +309,14 @@ func (s *Conn) cmdServerNlst(args string) { } if cnt == "" && s.path == "/" { - cnts, err := s.sw.ContainersAll(nil) + cnts, err := s.loadContainers() if err != nil { - fmt.Printf(err.Error()) - return + cnts, err = s.sw.ContainersAll(nil) + if err != nil { + fmt.Printf(err.Error()) + return + } + s.saveContainers(cnts) } files = append(files, ".", "..") for _, ct := range cnts { @@ -289,10 +328,14 @@ func (s *Conn) cmdServerNlst(args string) { if p != "." { opts.Path = p } - objs, err := s.sw.ObjectsAll(cnt, opts) + objs, err := s.loadObjects(cnt, opts) if err != nil { - fmt.Printf(err.Error()) - return + objs, err = s.sw.ObjectsAll(cnt, opts) + if err != nil { + fmt.Printf(err.Error()) + return + } + s.saveObjects(cnt, objs, opts) } fmt.Printf("%+v\n", objs) files = append(files, ".", "..") @@ -663,6 +706,7 @@ func (s *Conn) cmdServerRnfr(args string) { } func (s *Conn) cmdServerRnto(args string) { + fmt.Printf("cmdServerRnto: %s\n", args) var err error if args[0] == '/' { s.movedst = args @@ -673,6 +717,9 @@ func (s *Conn) cmdServerRnto(args string) { psrc := strings.Join(strings.Split(s.movesrc, "/")[2:], "/") cntdst := strings.Split(s.movedst, "/")[1] pdst := strings.Join(strings.Split(s.movedst, "/")[2:], "/") + + fmt.Printf("%s %s %s %s\n", cntsrc, psrc, cntdst, pdst) + if err = s.sw.ObjectMove(cntsrc, psrc, cntdst, pdst); err != nil { s.ctrl.PrintfLine("552 Failed to store file") return @@ -689,6 +736,10 @@ func (s *Conn) cmdServerAuth(args string) { } } +func (s *Conn) cmdServerHelp(args string) { + s.ctrl.PrintfLine(`200 Success`) +} + func (s *Conn) cmdServerNoop(args string) { s.ctrl.PrintfLine(`200 Success`) //SITE CHMOD 0644 /public/.wgetpaste.conf diff --git a/server.go b/server.go index d6a980c..ca98000 100644 --- a/server.go +++ b/server.go @@ -1,12 +1,25 @@ package main import ( + "database/sql" + "fmt" "net" "net/textproto" + _ "github.com/cznic/ql/driver" "github.com/ncw/swift" ) +var schema []string = []string{ + `CREATE TABLE IF NOT EXISTS containers (ID int, Name string, Bytes int, Count int)`, + `CREATE UNIQUE INDEX IF NOT EXISTS containersID ON containers (ID)`, + `CREATE INDEX IF NOT EXISTS containersName ON containers (Name)`, + `CREATE TABLE IF NOT EXISTS objects (ID int, Container string, Prefix string, ContentType string, LastModified time, Name string, Bytes int, Count int)`, + `CREATE UNIQUE INDEX IF NOT EXISTS objectsID ON objects (ID)`, + `CREATE INDEX IF NOT EXISTS objectsName ON objects (Name)`, + `CREATE INDEX IF NOT EXISTS objectsContainer ON objects (Container)`, +} + type Conn struct { ctrl *textproto.Conn data net.Conn @@ -22,12 +35,141 @@ type Conn struct { passive bool movesrc string movedst string + db *sql.DB } func (c *Conn) Close() error { + c.db.Close() return c.ctrl.Close() } func NewServer(c net.Conn) (*Conn, error) { - return &Conn{api: "https://api.clodo.ru", user: "storage_21_1", token: "56652e9028ded5ea5d4772ba80e578ce", ctrl: textproto.NewConn(c), path: "/"}, nil + var err error + conn := &Conn{api: "https://api.clodo.ru", user: "storage_21_1", token: "56652e9028ded5ea5d4772ba80e578ce", ctrl: textproto.NewConn(c), path: "/"} + conn.db, err = sql.Open("ql", "memory://mem.db") + if err != nil { + return nil, err + } + tx, err := conn.db.Begin() + if err != nil { + conn.db.Close() + return nil, err + } + for _, query := range schema { + if _, err := tx.Exec(query); err != nil { + conn.db.Close() + fmt.Printf("%s\n", err.Error()) + return nil, fmt.Errorf("q: %s e: %s", query, err.Error()) + } + } + + if err = tx.Commit(); err != nil { + return nil, err + } + + return conn, nil +} + +func (c *Conn) saveContainers(cnts []swift.Container) error { + fmt.Printf("%+v\n", cnts) + tx, err := c.db.Begin() + if err != nil { + fmt.Printf("777 %s\n", err.Error()) + return err + } + + for _, cnt := range cnts { + if _, err = tx.Exec(`INSERT INTO containers (Name, Bytes, Count) VALUES ($1, $2, $3)`, cnt.Name, cnt.Bytes, cnt.Count); err != nil { + fmt.Printf("9999 %s\n", err.Error()) + return err + } + } + + if err = tx.Commit(); err != nil { + fmt.Printf("ssss %s\n", err.Error()) + return err + } + + return nil +} + +func (c *Conn) saveObjects(cnt string, objs []swift.Object, opts *swift.ObjectsOpts) error { + fmt.Printf("%+v\n", objs) + tx, err := c.db.Begin() + if err != nil { + fmt.Printf("%s\n", err.Error()) + return err + } + + for _, obj := range objs { + if _, err = tx.Exec(`INSERT INTO objects (Container, Prefix, ContentType, LastModified, Name, Bytes, Count) VALUES ($1, $2, $3, $4, $5, $6, $7)`, cnt, opts.Prefix, obj.ContentType, obj.LastModified, obj.Name, obj.Bytes, 1); err != nil { + fmt.Printf("aaaa %s\n", err.Error()) + return err + } + } + + if err = tx.Commit(); err != nil { + fmt.Printf("%s\n", err.Error()) + return err + } + return nil +} + +func (c *Conn) loadObjects(cnt string, opts *swift.ObjectsOpts) ([]swift.Object, error) { + var objs []swift.Object + var res swift.Object + + result, err := c.db.Query(`SELECT ContentType, LastModified, Name, Bytes from objects where Container==$1 && Prefix==$1`, cnt, opts.Prefix) + if err != nil { + fmt.Printf("bbb %s\n", err.Error()) + return objs, err + } + defer result.Close() + + for result.Next() { + if err = result.Scan(&res.ContentType, &res.LastModified, &res.Name, &res.Bytes); err != nil { + fmt.Printf("%s\n", err.Error()) + return objs, err + } + objs = append(objs, res) + } + + if err := result.Err(); err != nil { + fmt.Printf("%s\n", err.Error()) + return objs, err + } + if len(objs) < 1 { + return objs, fmt.Errorf("empty") + } + return objs, nil +} + +func (c *Conn) loadContainers() ([]swift.Container, error) { + var cnts []swift.Container + var res swift.Container + + result, err := c.db.Query(`SELECT Name, Bytes, Count from containers`) + if err != nil { + fmt.Printf("111 %s\n", err.Error()) + return cnts, err + } + defer result.Close() + + for result.Next() { + if err = result.Scan(&res.Name, &res.Bytes, &res.Count); err != nil { + fmt.Printf("222 %s\n", err.Error()) + return cnts, err + } + cnts = append(cnts, res) + } + + if err := result.Err(); err != nil { + fmt.Printf("333 %s\n", err.Error()) + return cnts, err + } + if len(cnts) < 1 { + return cnts, fmt.Errorf("empty") + } + fmt.Printf("444 %+v\n", cnts) + return cnts, nil }