unistack-org/blob-ftp
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement simple in memory caching

Browse Source 
closes #13

Signed-off-by: Vasiliy Tolstov <v.tolstov@selfip.ru>
This commit is contained in:
Василий Толстов 2015-05-07 14:52:59 +03:00
parent 066a8d5b6c
commit 797f9d11e7
2 changed files with 210 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
main.go
View File

@ -40,6 +40,14 @@ func handle(c net.Conn) {
if err != nil {
fmt.Printf(err.Error())
}
/*
s.cache = groupcache.NewGroup("cache", 10 * 1024, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
contents, err := ioutil.ReadFile(key)
dest.SetBytes(contents)
return err
}))
*/
s.ctrl.PrintfLine("220 %s [%s]", "go-ftp", c.LocalAddr())
for {
line, err := s.ctrl.ReadLine()
@ -59,6 +67,8 @@ func handle(c net.Conn) {
s.cmdServerCdup(params)
case "RNFR":
s.cmdServerRnfr(params)
case "RNTO":
s.cmdServerRnto(params)
case "OPTS":
s.cmdServerOpts(params)
case "MDTM":
@ -87,6 +97,8 @@ func handle(c net.Conn) {
s.cmdServerQuit(params)
s.Close()
return
case "HELP":
s.cmdServerHelp(params)
case "SIZE":
s.cmdServerSize(params)
case "CWD", "XCWD":
@ -191,21 +203,34 @@ func (s *Conn) cmdServerList(args string) {
} else {
if s.path != "/" {
cnt = strings.Split(s.path, "/")[1]
if len(args) > 0 {
p = args
} else {
p = filepath.Clean(strings.Join(strings.Split(s.path, "/")[2:], "/"))
}
fmt.Printf("cnt1 path %s cnt %s p %s\n", s.path, cnt, p)
} else {
fmt.Printf("cnt2 %s\n", s.path)
if len(args) > 0 {
cnt = strings.Split(args, "/")[0]
p = strings.Join(strings.Split(args, "/")[1:], "/")
} else {
cnt = strings.Split(s.path, "/")[1]
p = filepath.Clean(strings.Join(strings.Split(s.path, "/")[2:], "/"))
}
}
}
fmt.Printf("cnt3 %s p %s\n", cnt, p)
if cnt == "" {
cnts, err := s.sw.ContainersAll(nil)
cnts, err := s.loadContainers()
if err != nil {
cnts, err = s.sw.ContainersAll(nil)
if err != nil {
fmt.Printf(err.Error())
return
}
s.saveContainers(cnts)
}
files = append(files, NewDirItem(".", 4096, 0), NewDirItem("..", 4096, 0))
for _, ct := range cnts {
it := NewDirItem(ct.Name, ct.Bytes, ct.Count)
@ -218,18 +243,28 @@ func (s *Conn) cmdServerList(args string) {
opts := &swift.ObjectsOpts{Delimiter: '/'}
fmt.Printf("cnt4: %s p: %s\n", cnt, p)
if p != "." {
opts.Path = p
opts.Prefix = p
if !strings.HasSuffix(opts.Prefix, "/") {
opts.Prefix += "/"
}
objs, err := s.sw.ObjectsAll(cnt, opts)
}
objs, err := s.loadObjects(cnt, opts)
if err != nil {
objs, err = s.sw.ObjectsAll(cnt, opts)
if err != nil {
fmt.Printf(err.Error())
return
}
s.saveObjects(cnt, objs, opts)
}
fmt.Printf("%+v\n", objs)
files = append(files, NewDirItem(".", 4096, 0), NewDirItem("..", 4096, 0))
var it os.FileInfo
for _, obj := range objs {
if obj.SubDir != "" || strings.Index(obj.Name, "/") > 0 {
continue
}
if obj.PseudoDirectory || obj.ContentType == "application/directory" {
it = NewDirItem(obj.Name, obj.Bytes, 1)
} else {
@ -274,11 +309,15 @@ func (s *Conn) cmdServerNlst(args string) {
}
if cnt == "" && s.path == "/" {
cnts, err := s.sw.ContainersAll(nil)
cnts, err := s.loadContainers()
if err != nil {
cnts, err = s.sw.ContainersAll(nil)
if err != nil {
fmt.Printf(err.Error())
return
}
s.saveContainers(cnts)
}
files = append(files, ".", "..")
for _, ct := range cnts {
files = append(files, ct.Name)
@ -289,11 +328,15 @@ func (s *Conn) cmdServerNlst(args string) {
if p != "." {
opts.Path = p
}
objs, err := s.sw.ObjectsAll(cnt, opts)
objs, err := s.loadObjects(cnt, opts)
if err != nil {
objs, err = s.sw.ObjectsAll(cnt, opts)
if err != nil {
fmt.Printf(err.Error())
return
}
s.saveObjects(cnt, objs, opts)
}
fmt.Printf("%+v\n", objs)
files = append(files, ".", "..")
@ -663,6 +706,7 @@ func (s *Conn) cmdServerRnfr(args string) {
}
func (s *Conn) cmdServerRnto(args string) {
fmt.Printf("cmdServerRnto: %s\n", args)
var err error
if args[0] == '/' {
s.movedst = args
@ -673,6 +717,9 @@ func (s *Conn) cmdServerRnto(args string) {
psrc := strings.Join(strings.Split(s.movesrc, "/")[2:], "/")
cntdst := strings.Split(s.movedst, "/")[1]
pdst := strings.Join(strings.Split(s.movedst, "/")[2:], "/")
fmt.Printf("%s %s %s %s\n", cntsrc, psrc, cntdst, pdst)
if err = s.sw.ObjectMove(cntsrc, psrc, cntdst, pdst); err != nil {
s.ctrl.PrintfLine("552 Failed to store file")
return
@ -689,6 +736,10 @@ func (s *Conn) cmdServerAuth(args string) {
}
}
func (s *Conn) cmdServerHelp(args string) {
s.ctrl.PrintfLine(`200 Success`)
}
func (s *Conn) cmdServerNoop(args string) {
s.ctrl.PrintfLine(`200 Success`)
//SITE CHMOD 0644 /public/.wgetpaste.conf

144
server.go
View File

@ -1,12 +1,25 @@
package main
import (
"database/sql"
"fmt"
"net"
"net/textproto"
_ "github.com/cznic/ql/driver"
"github.com/ncw/swift"
)
var schema []string = []string{
`CREATE TABLE IF NOT EXISTS containers (ID int, Name string, Bytes int, Count int)`,
`CREATE UNIQUE INDEX IF NOT EXISTS containersID ON containers (ID)`,
`CREATE INDEX IF NOT EXISTS containersName ON containers (Name)`,
`CREATE TABLE IF NOT EXISTS objects (ID int, Container string, Prefix string, ContentType string, LastModified time, Name string, Bytes int, Count int)`,
`CREATE UNIQUE INDEX IF NOT EXISTS objectsID ON objects (ID)`,
`CREATE INDEX IF NOT EXISTS objectsName ON objects (Name)`,
`CREATE INDEX IF NOT EXISTS objectsContainer ON objects (Container)`,
}
type Conn struct {
ctrl *textproto.Conn
data net.Conn
@ -22,12 +35,141 @@ type Conn struct {
passive bool
movesrc string
movedst string
db *sql.DB
}
func (c *Conn) Close() error {
c.db.Close()
return c.ctrl.Close()
}
func NewServer(c net.Conn) (*Conn, error) {
return &Conn{api: "https://api.clodo.ru", user: "storage_21_1", token: "56652e9028ded5ea5d4772ba80e578ce", ctrl: textproto.NewConn(c), path: "/"}, nil
var err error
conn := &Conn{api: "https://api.clodo.ru", user: "storage_21_1", token: "56652e9028ded5ea5d4772ba80e578ce", ctrl: textproto.NewConn(c), path: "/"}
conn.db, err = sql.Open("ql", "memory://mem.db")
if err != nil {
return nil, err
}
tx, err := conn.db.Begin()
if err != nil {
conn.db.Close()
return nil, err
}
for _, query := range schema {
if _, err := tx.Exec(query); err != nil {
conn.db.Close()
fmt.Printf("%s\n", err.Error())
return nil, fmt.Errorf("q: %s e: %s", query, err.Error())
}
}
if err = tx.Commit(); err != nil {
return nil, err
}
return conn, nil
}
func (c *Conn) saveContainers(cnts []swift.Container) error {
fmt.Printf("%+v\n", cnts)
tx, err := c.db.Begin()
if err != nil {
fmt.Printf("777 %s\n", err.Error())
return err
}
for _, cnt := range cnts {
if _, err = tx.Exec(`INSERT INTO containers (Name, Bytes, Count) VALUES ($1, $2, $3)`, cnt.Name, cnt.Bytes, cnt.Count); err != nil {
fmt.Printf("9999 %s\n", err.Error())
return err
}
}
if err = tx.Commit(); err != nil {
fmt.Printf("ssss %s\n", err.Error())
return err
}
return nil
}
func (c *Conn) saveObjects(cnt string, objs []swift.Object, opts *swift.ObjectsOpts) error {
fmt.Printf("%+v\n", objs)
tx, err := c.db.Begin()
if err != nil {
fmt.Printf("%s\n", err.Error())
return err
}
for _, obj := range objs {
if _, err = tx.Exec(`INSERT INTO objects (Container, Prefix, ContentType, LastModified, Name, Bytes, Count) VALUES ($1, $2, $3, $4, $5, $6, $7)`, cnt, opts.Prefix, obj.ContentType, obj.LastModified, obj.Name, obj.Bytes, 1); err != nil {
fmt.Printf("aaaa %s\n", err.Error())
return err
}
}
if err = tx.Commit(); err != nil {
fmt.Printf("%s\n", err.Error())
return err
}
return nil
}
func (c *Conn) loadObjects(cnt string, opts *swift.ObjectsOpts) ([]swift.Object, error) {
var objs []swift.Object
var res swift.Object
result, err := c.db.Query(`SELECT ContentType, LastModified, Name, Bytes from objects where Container==$1 && Prefix==$1`, cnt, opts.Prefix)
if err != nil {
fmt.Printf("bbb %s\n", err.Error())
return objs, err
}
defer result.Close()
for result.Next() {
if err = result.Scan(&res.ContentType, &res.LastModified, &res.Name, &res.Bytes); err != nil {
fmt.Printf("%s\n", err.Error())
return objs, err
}
objs = append(objs, res)
}
if err := result.Err(); err != nil {
fmt.Printf("%s\n", err.Error())
return objs, err
}
if len(objs) < 1 {
return objs, fmt.Errorf("empty")
}
return objs, nil
}
func (c *Conn) loadContainers() ([]swift.Container, error) {
var cnts []swift.Container
var res swift.Container
result, err := c.db.Query(`SELECT Name, Bytes, Count from containers`)
if err != nil {
fmt.Printf("111 %s\n", err.Error())
return cnts, err
}
defer result.Close()
for result.Next() {
if err = result.Scan(&res.Name, &res.Bytes, &res.Count); err != nil {
fmt.Printf("222 %s\n", err.Error())
return cnts, err
}
cnts = append(cnts, res)
}
if err := result.Err(); err != nil {
fmt.Printf("333 %s\n", err.Error())
return cnts, err
}
if len(cnts) < 1 {
return cnts, fmt.Errorf("empty")
}
fmt.Printf("444 %+v\n", cnts)
return cnts, nil
}