Merge pull request #63 from YorikSar/streams

Add streams support to libvirt RPC API

AUTHORS

@@ -17,3 +17,4 @@ Simarpreet Singh <simar@linux.com>
 Alexander Polyakov <apolyakov@beget.com>
 Amanda Andrade <amanda.andrade@serpro.gov.br>
 Geoff Hickey <ghickey@digitalocean.com>
+Yuriy Taraday <yorik.sar@gmail.com>

internal/lvgen/generate.go

@@ -166,6 +166,14 @@ type Proc struct {
 	Ret            []Decl // The contents of the ret struct for this procedure.
 	ArgsStruct     string // The name of the args struct for this procedure.
 	RetStruct      string // The name of the ret struct for this procedure.
+	ReadStreamIdx  int    // The index of read stream in function argument list
+	WriteStreamIdx int    // The index of read stream in function argument list
+}
+
+// ProcMeta holds information from annotations attached to a libvirt procedure
+type ProcMeta struct {
+	ReadStream  int
+	WriteStream int
 }
 
 type structStack []*Structure
@@ -552,7 +560,31 @@ func AddEnumVal(name, val string) error {
 	if err != nil {
 		return fmt.Errorf("invalid enum value %v = %v", name, val)
 	}
-	return addEnumVal(name, ev)
+	return addEnumVal(name, ev, nil)
+}
+
+// AddEnumValMeta will add a new enum value with attached metadata to the list.
+// Metadata is parsed from annotations in libvirt RPC description file that are
+// in block comment preceding every function in enum, it looks like this:
+// /**
+//  * @generate: both
+//  * @readstream: 1
+//  * @sparseflag: VIR_STORAGE_VOL_DOWNLOAD_SPARSE_STREAM
+//  * @acl: storage_vol:data_read
+//  */
+// See full description of possible annotations in libvirt's src/remote/remote_protocol.x
+// at the top of remote_procedure enum.
+// We're parsing only @readstream and @writestream annotations at the moment.
+func AddEnumValMeta(name, val, meta string) error {
+	ev, err := parseNumber(val)
+	if err != nil {
+		return fmt.Errorf("invalid enum value %v = %v", name, val)
+	}
+	metaObj, err := parseMeta(meta)
+	if err != nil {
+		return fmt.Errorf("invalid metadata for enum value %v: %v", name, err)
+	}
+	return addEnumVal(name, ev, metaObj)
 }
 
 // AddEnumAutoVal adds an enum to the list, using the automatically-incremented
@@ -560,14 +592,14 @@ func AddEnumVal(name, val string) error {
 // explicit value.
 func AddEnumAutoVal(name string) error {
 	CurrentEnumVal++
-	return addEnumVal(name, CurrentEnumVal)
+	return addEnumVal(name, CurrentEnumVal, nil)
 }
 
-func addEnumVal(name string, val int64) error {
+func addEnumVal(name string, val int64, meta *ProcMeta) error {
 	goname := constNameTransform(name)
 	Gen.EnumVals = append(Gen.EnumVals, ConstItem{goname, name, fmt.Sprintf("%d", val)})
 	CurrentEnumVal = val
-	addProc(goname, name, val)
+	addProc(goname, name, val, meta)
 	return nil
 }
 
@@ -584,12 +616,16 @@ func AddConst(name, val string) error {
 
 // addProc checks an enum value to see if it's a procedure number. If so, we
 // add the procedure to our list for later generation.
-func addProc(goname, lvname string, val int64) {
+func addProc(goname, lvname string, val int64, meta *ProcMeta) {
 	if !strings.HasPrefix(goname, "Proc") {
 		return
 	}
 	goname = goname[4:]
-	proc := &Proc{Num: val, Name: goname, LVName: lvname}
+	proc := &Proc{Num: val, Name: goname, LVName: lvname, ReadStreamIdx: -1, WriteStreamIdx: -1}
+	if meta != nil {
+		proc.ReadStreamIdx = meta.ReadStream
+		proc.WriteStreamIdx = meta.WriteStream
+	}
 	Gen.Procs = append(Gen.Procs, *proc)
 }
 
@@ -605,6 +641,41 @@ func parseNumber(val string) (int64, error) {
 	return n, err
 }
 
+// parseMeta parses procedure metadata to simple string mapping
+func parseMeta(meta string) (*ProcMeta, error) {
+	res := &ProcMeta{
+		ReadStream:  -1,
+		WriteStream: -1,
+	}
+	for _, line := range strings.Split(meta, "\n") {
+		atInd := strings.Index(line, "@")
+		if atInd == -1 {
+			// Should be only first and last line of comment
+			continue
+		}
+		spl := strings.SplitN(line[atInd+1:], ":", 2)
+		if len(spl) != 2 {
+			return nil, fmt.Errorf("invalid annotation: %s", meta)
+		}
+		spl[1] = strings.Trim(spl[1], " ")
+		switch spl[0] {
+		case "readstream":
+			var err error
+			res.ReadStream, err = strconv.Atoi(spl[1])
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for readstream: %s", spl[1])
+			}
+		case "writestream":
+			var err error
+			res.WriteStream, err = strconv.Atoi(spl[1])
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for writestream: %s", spl[1])
+			}
+		}
+	}
+	return res, nil
+}
+
 // StartStruct is called from the parser when a struct definition is found, but
 // before the member declarations are processed.
 func StartStruct(name string) {

internal/lvgen/lvlexer.go

@@ -259,12 +259,19 @@ func lexText(l *Lexer) stateFn {
 
 // lexBlockComment is used when we find a comment marker '/*' in the input.
 func lexBlockComment(l *Lexer) stateFn {
+	// Double star is used only at the start of metadata comments
+	metadataComment := strings.HasPrefix(l.input[l.pos:], "/**")
 	for {
 		if strings.HasPrefix(l.input[l.pos:], "*/") {
-			// Found the end. Advance past the '*/' and discard the comment body.
+			// Found the end. Advance past the '*/' and discard the comment body
+			// unless it's a metadata comment
 			l.next()
 			l.next()
+			if metadataComment {
+				l.emit(METADATACOMMENT)
+			} else {
 				l.ignore()
+			}
 			return lexText
 		}
 		if l.next() == eof {

internal/lvgen/procedures.tmpl

@@ -23,6 +23,7 @@ package libvirt
 import (
 	"bytes"
 	"fmt"
+	"io"
 
 	"github.com/davecgh/go-xdr/xdr2"
 	"github.com/digitalocean/go-libvirt/internal/constants"
@@ -126,9 +127,15 @@ func decodeTypedParams(dec *xdr.Decoder) ([]TypedParam, error) {
 	return params, nil
 }
 
-{{range .Procs}}
+{{range $proc := .Procs}}
 // {{.Name}} is the go wrapper for {{.LVName}}.
-func (l *Libvirt) {{.Name}}({{range $ix, $arg := .Args}}{{if $ix}}, {{end}}{{.Name}} {{.Type}}{{end}}) ({{range .Ret}}r{{.Name}} {{.Type}}, {{end}}err error) {
+func (l *Libvirt) {{.Name}}(
+  {{- range $ix, $arg := .Args}}
+    {{- if (eq $ix $proc.WriteStreamIdx)}}{{if $ix}}, {{end}}outStream io.Reader{{end}}
+    {{- if (eq $ix $proc.ReadStreamIdx)}}{{if $ix}}, {{end}}inStream io.Writer{{end}}
+    {{- if $ix}}, {{end}}{{.Name}} {{.Type}}
+  {{- end -}}
+) ({{range .Ret}}r{{.Name}} {{.Type}}, {{end}}err error) {
 	var buf []byte
 {{if .ArgsStruct}}
 	args := {{.ArgsStruct}} {
@@ -141,7 +148,10 @@ func (l *Libvirt) {{.Name}}({{range $ix, $arg := .Args}}{{if $ix}}, {{end}}{{.Na
 	}
 {{end}}
 {{if .RetStruct}}	var r response{{end}}
-	{{if .RetStruct}}r{{else}}_{{end}}, err = l.request({{.Num}}, constants.Program, buf)
+	{{if .RetStruct}}r{{else}}_{{end}}, err = l.requestStream({{.Num}}, constants.Program, buf,
+    {{- if (ne .WriteStreamIdx -1)}} outStream,{{else}} nil,{{end}}
+    {{- if (ne .ReadStreamIdx -1)}} inStream{{else}} nil{{end -}}
+    )
 	if err != nil {
 		return
 	}

internal/lvgen/sunrpc.y

@@ -56,6 +56,7 @@ import (
 %token IDENTIFIER CONSTANT ERROR
 // RPCL additional tokens:
 %token PROGRAM VERSION
+%token METADATACOMMENT
 
 %%
 
@@ -99,6 +100,13 @@ enum_value
             return 1
         }
     }
+    | METADATACOMMENT enum_value_ident '=' value {
+        err := AddEnumValMeta($2.val, $4.val, $1.val)
+        if err != nil {
+            yylex.Error(err.Error())
+            return 1
+        }
+    }
     | enum_value_ident '=' value {
         err := AddEnumVal($1.val, $3.val)
         if err != nil {

internal/lvgen/y.go

@@ -59,6 +59,7 @@ const CONSTANT = 57366
 const ERROR = 57367
 const PROGRAM = 57368
 const VERSION = 57369
+const METADATACOMMENT = 57370
 
 var yyToknames = [...]string{
 	"$end",
@@ -88,6 +89,7 @@ var yyToknames = [...]string{
 	"ERROR",
 	"PROGRAM",
 	"VERSION",
+	"METADATACOMMENT",
 	"';'",
 	"'{'",
 	"'}'",
@@ -108,7 +110,7 @@ const yyEofCode = 1
 const yyErrCode = 2
 const yyInitialStackSize = 16
 
-//line sunrpc.y:259
+//line sunrpc.y:267
 
 //line yacctab:1
 var yyExca = [...]int{
@@ -119,106 +121,107 @@ var yyExca = [...]int{
 
 const yyPrivate = 57344
 
-const yyLast = 148
+const yyLast = 153
 
 var yyAct = [...]int{
 
-	84, 77, 36, 111, 103, 76, 61, 67, 32, 129,
+	86, 79, 36, 115, 107, 78, 62, 32, 68, 55,
-	55, 126, 128, 100, 37, 117, 85, 86, 78, 63,
+	57, 133, 130, 132, 104, 121, 87, 88, 80, 64,
-	98, 97, 73, 31, 74, 69, 131, 115, 71, 89,
+	37, 102, 75, 31, 76, 101, 114, 135, 119, 72,
-	41, 94, 72, 62, 40, 10, 39, 43, 42, 13,
+	92, 41, 97, 89, 63, 40, 10, 39, 43, 42,
-	30, 118, 14, 38, 107, 48, 49, 50, 51, 47,
+	13, 73, 30, 14, 38, 122, 48, 49, 50, 51,
-	90, 11, 79, 64, 10, 70, 106, 96, 13, 54,
+	47, 111, 93, 81, 71, 11, 110, 138, 10, 65,
-	12, 14, 52, 29, 134, 127, 119, 108, 91, 75,
+	99, 54, 13, 52, 12, 14, 29, 131, 123, 74,
-	80, 15, 16, 110, 87, 88, 85, 86, 59, 60,
+	77, 112, 94, 59, 82, 15, 90, 91, 58, 70,
-	63, 93, 83, 48, 49, 50, 51, 92, 95, 58,
+	16, 64, 85, 96, 87, 88, 60, 61, 95, 84,
-	82, 27, 25, 23, 20, 102, 18, 99, 2, 105,
+	100, 98, 48, 49, 50, 51, 113, 59, 106, 27,
-	101, 46, 8, 66, 45, 7, 109, 44, 4, 104,
+	103, 25, 109, 105, 23, 20, 18, 67, 46, 8,
-	113, 105, 114, 116, 120, 28, 122, 81, 8, 68,
+	108, 45, 7, 44, 4, 109, 2, 120, 124, 117,
-	123, 7, 26, 124, 4, 121, 125, 112, 130, 53,
+	126, 118, 83, 69, 127, 8, 26, 128, 7, 125,
-	24, 132, 133, 65, 22, 35, 34, 33, 21, 19,
+	4, 129, 134, 28, 116, 136, 137, 53, 24, 66,
-	57, 56, 17, 9, 6, 5, 3, 1,
+	22, 35, 34, 33, 21, 19, 56, 17, 9, 6,
+	5, 3, 1,
 }
 var yyPact = [...]int{
 
-	45, -1000, -1000, 44, -1000, -1000, -1000, -1000, -1000, -1000,
+	49, -1000, -1000, 51, -1000, -1000, -1000, -1000, -1000, -1000,
-	73, 71, -1000, 70, 69, 68, 45, 34, -1000, 8,
+	83, 82, -1000, 81, 78, 76, 49, 36, -1000, 9,
-	-1000, 26, 33, -1000, -1000, -1000, 30, -1000, -1000, 66,
+	-1000, 27, 33, -1000, -1000, -1000, 31, -1000, -1000, 50,
-	55, -1000, -1000, -1000, -1000, -1000, -4, -1000, 64, -1000,
+	63, -1000, -1000, -1000, -1000, -1000, -4, -1000, 73, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-	-1000, -1000, -1000, 89, -2, 25, -3, 0, -1000, -1000,
+	-1000, -1000, -1000, 93, 52, 23, -3, 8, 74, -1000,
-	-1000, -11, 57, -1000, -1000, 26, -20, 22, 42, 67,
+	-1000, -1000, -12, 58, -1000, -1000, 27, -21, 22, 45,
-	-1000, 66, 53, 53, -7, -1000, 20, 40, 26, -1,
+	66, -1000, 50, 61, 0, 61, -7, -1000, 21, 43,
-	-2, 28, -1000, -1000, -1000, -1000, -1000, -13, -16, -1000,
+	27, -1, 52, 30, -1000, -1000, -1000, -1000, -1000, 61,
-	-1000, 26, -26, 57, 53, -1000, 26, -1000, -1000, -1000,
+	-10, -16, -1000, -1000, 27, -26, 58, 61, -1000, 27,
-	27, -1000, -1000, 14, 39, 50, 105, -5, 26, -23,
+	-1000, -1000, -1000, -1000, 26, -1000, -1000, 20, 42, 3,
-	-1000, 11, 38, 53, -1000, 53, -1000, 26, -1000, 105,
+	114, -5, 27, -24, -1000, 14, 39, 61, -1000, 61,
-	-1000, -29, 37, -27, -1000, -31, 26, -1000, -6, 26,
+	-1000, 27, -1000, 114, -1000, -29, 38, -27, -1000, -30,
-	-1000, 53, -1000, 36, -1000,
+	27, -1000, -6, 27, -1000, 61, -1000, 28, -1000,
 }
 var yyPgo = [...]int{
 
-	0, 147, 98, 0, 146, 107, 145, 144, 104, 101,
+	0, 152, 116, 0, 151, 113, 150, 149, 111, 108,
-	143, 142, 10, 141, 140, 139, 138, 1, 8, 137,
+	148, 147, 9, 146, 10, 145, 144, 1, 7, 143,
-	136, 135, 2, 6, 14, 134, 133, 5, 130, 129,
+	142, 141, 2, 6, 20, 140, 139, 5, 138, 137,
-	3, 127, 126, 125, 122, 7, 119, 117, 4, 109,
+	3, 134, 131, 129, 126, 8, 123, 122, 4, 110,
-	106,
+	96,
 }
 var yyR1 = [...]int{
 
 	0, 1, 3, 3, 2, 2, 4, 4, 4, 4,
-	4, 4, 5, 12, 12, 13, 13, 11, 14, 6,
+	4, 4, 5, 12, 12, 13, 13, 13, 11, 14,
-	6, 15, 16, 7, 17, 17, 17, 17, 18, 22,
+	6, 6, 15, 16, 7, 17, 17, 17, 17, 18,
 	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
-	24, 24, 24, 24, 23, 19, 20, 20, 21, 26,
+	22, 24, 24, 24, 24, 23, 19, 20, 20, 21,
-	8, 25, 27, 27, 29, 9, 28, 30, 30, 32,
+	26, 8, 25, 27, 27, 29, 9, 28, 30, 30,
-	31, 33, 31, 10, 34, 35, 35, 36, 37, 38,
+	32, 31, 33, 31, 10, 34, 35, 35, 36, 37,
-	38, 39, 40,
+	38, 38, 39, 40,
 }
 var yyR2 = [...]int{
 
 	0, 1, 1, 1, 2, 3, 1, 1, 1, 1,
-	1, 1, 5, 1, 3, 1, 3, 1, 1, 4,
+	1, 1, 5, 1, 3, 1, 4, 3, 1, 1,
-	4, 1, 0, 3, 1, 1, 1, 1, 2, 1,
+	4, 4, 1, 0, 3, 1, 1, 1, 1, 2,
-	2, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 2, 1, 1, 1, 1, 1, 1, 1, 1,
-	1, 1, 1, 1, 1, 5, 5, 4, 3, 0,
+	1, 1, 1, 1, 1, 1, 5, 5, 4, 3,
-	6, 1, 2, 3, 0, 10, 1, 2, 3, 0,
+	0, 6, 1, 2, 3, 0, 10, 1, 2, 3,
-	5, 0, 4, 7, 1, 2, 3, 8, 1, 2,
+	0, 5, 0, 4, 7, 1, 2, 3, 8, 1,
-	3, 8, 1,
+	2, 3, 8, 1,
 }
 var yyChk = [...]int{
 
 	-1000, -1, -2, -4, -5, -6, -7, -8, -9, -10,
-	9, 6, 15, 13, 16, 26, 28, -11, 23, -15,
+	9, 6, 15, 13, 16, 26, 29, -11, 23, -15,
-	23, -16, -25, 23, -28, 23, -34, 23, -2, 29,
+	23, -16, -25, 23, -28, 23, -34, 23, -2, 30,
-	32, -17, -18, -19, -20, -21, -22, -24, 17, 10,
+	33, -17, -18, -19, -20, -21, -22, -24, 17, 10,
 	8, 4, 12, 11, -5, -8, -9, 23, 19, 20,
-	21, 22, 29, -29, 29, -12, -13, -14, 23, 23,
+	21, 22, 30, -29, 30, -12, -13, -14, 28, 23,
-	24, -23, 37, 23, -24, -26, 14, -35, -36, 27,
+	23, 24, -23, 38, 23, -24, -26, 14, -35, -36,
-	30, 31, 32, 33, 35, -23, -27, -17, 38, 30,
+	27, 31, 32, 33, -14, 34, 36, -23, -27, -17,
-	28, -37, 23, -12, -3, 23, 24, -3, -3, 36,
+	39, 31, 29, -37, 23, -12, -3, 23, 24, 33,
-	30, 28, -18, -22, 32, -35, 29, 34, 36, -27,
+	-3, -3, 37, 31, 29, -18, -22, 33, -35, 30,
-	39, -23, -3, -38, -39, -22, 29, 30, 28, -40,
+	-3, 35, 37, -27, 40, -23, -3, -38, -39, -22,
-	23, -30, -31, 5, 7, 32, -38, 38, 30, 28,
+	30, 31, 29, -40, 23, -30, -31, 5, 7, 33,
-	-3, -33, -3, -22, -30, -32, 40, 28, 39, 40,
+	-38, 39, 31, 29, -3, -33, -3, -22, -30, -32,
-	-17, 32, -17, -3, 28,
+	41, 29, 40, 41, -17, 33, -17, -3, 29,
 }
 var yyDef = [...]int{
 
 	0, -2, 1, 0, 6, 7, 8, 9, 10, 11,
-	0, 0, 22, 0, 0, 0, 4, 0, 17, 0,
+	0, 0, 23, 0, 0, 0, 4, 0, 18, 0,
-	21, 0, 0, 51, 54, 56, 0, 64, 5, 0,
+	22, 0, 0, 52, 55, 57, 0, 65, 5, 0,
-	0, 23, 24, 25, 26, 27, 0, 29, 0, 31,
+	0, 24, 25, 26, 27, 28, 0, 30, 0, 32,
-	32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
+	33, 34, 35, 36, 37, 38, 39, 40, 41, 42,
-	42, 43, 49, 0, 0, 0, 13, 15, 18, 19,
+	43, 44, 50, 0, 0, 0, 13, 15, 0, 19,
-	20, 28, 0, 44, 30, 0, 0, 0, 0, 0,
+	20, 21, 29, 0, 45, 31, 0, 0, 0, 0,
-	12, 0, 0, 0, 0, 48, 0, 0, 0, 0,
+	0, 12, 0, 0, 0, 0, 0, 49, 0, 0,
-	65, 0, 68, 14, 16, 2, 3, 0, 0, 47,
+	0, 0, 66, 0, 69, 14, 17, 2, 3, 0,
-	50, 52, 0, 0, 0, 66, 0, 45, 46, 53,
+	0, 0, 48, 51, 53, 0, 0, 0, 67, 0,
-	0, 28, 63, 0, 0, 0, 0, 0, 69, 0,
+	16, 46, 47, 54, 0, 29, 64, 0, 0, 0,
-	72, 0, 0, 0, 61, 0, 70, 0, 55, 57,
+	0, 0, 70, 0, 73, 0, 0, 0, 62, 0,
-	59, 0, 0, 0, 58, 0, 0, 67, 0, 0,
+	71, 0, 56, 58, 60, 0, 0, 0, 59, 0,
-	62, 0, 60, 0, 71,
+	0, 68, 0, 0, 63, 0, 61, 0, 72,
 }
 var yyTok1 = [...]int{
 
@@ -226,21 +229,21 @@ var yyTok1 = [...]int{
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	38, 39, 37, 3, 31, 3, 3, 3, 3, 3,
+	39, 40, 38, 3, 32, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 40, 28,
+	3, 3, 3, 3, 3, 3, 3, 3, 41, 29,
-	35, 32, 36, 3, 3, 3, 3, 3, 3, 3,
+	36, 33, 37, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 33, 3, 34, 3, 3, 3, 3, 3, 3,
+	3, 34, 3, 35, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 29, 3, 30,
+	3, 3, 3, 30, 3, 31,
 }
 var yyTok2 = [...]int{
 
 	2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
 	12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
-	22, 23, 24, 25, 26, 27,
+	22, 23, 24, 25, 26, 27, 28,
 }
 var yyTok3 = [...]int{
 	0,
@@ -585,13 +588,13 @@ yydefault:
 
 	case 12:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line sunrpc.y:86
+		//line sunrpc.y:87
 		{
 			StartEnum(yyDollar[2].val)
 		}
 	case 15:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:95
+		//line sunrpc.y:96
 		{
 			err := AddEnumAutoVal(yyDollar[1].val)
 			if err != nil {
@@ -600,8 +603,18 @@ yydefault:
 			}
 		}
 	case 16:
+		yyDollar = yyS[yypt-4 : yypt+1]
+		//line sunrpc.y:103
+		{
+			err := AddEnumValMeta(yyDollar[2].val, yyDollar[4].val, yyDollar[1].val)
+			if err != nil {
+				yylex.Error(err.Error())
+				return 1
+			}
+		}
+	case 17:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line sunrpc.y:102
+		//line sunrpc.y:110
 		{
 			err := AddEnumVal(yyDollar[1].val, yyDollar[3].val)
 			if err != nil {
@@ -609,9 +622,9 @@ yydefault:
 				return 1
 			}
 		}
-	case 20:
+	case 21:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line sunrpc.y:124
+		//line sunrpc.y:132
 		{
 			err := AddConst(yyDollar[2].val, yyDollar[4].val)
 			if err != nil {
@@ -619,147 +632,147 @@ yydefault:
 				return 1
 			}
 		}
-	case 22:
+	case 23:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:138
+		//line sunrpc.y:146
 		{
 			StartTypedef()
 		}
-	case 28:
+	case 29:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line sunrpc.y:149
+		//line sunrpc.y:157
 		{
 			AddDeclaration(yyDollar[2].val, yyDollar[1].val)
 		}
-	case 30:
+	case 31:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line sunrpc.y:154
+		//line sunrpc.y:162
 		{
 			yyVAL.val = "u" + yyDollar[2].val
 		}
-	case 31:
+	case 32:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:155
+		//line sunrpc.y:163
 		{
 			yyVAL.val = "float32"
 		}
-	case 32:
+	case 33:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:156
+		//line sunrpc.y:164
 		{
 			yyVAL.val = "float64"
 		}
-	case 33:
+	case 34:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:157
+		//line sunrpc.y:165
 		{
 			yyVAL.val = "bool"
 		}
-	case 34:
+	case 35:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:158
+		//line sunrpc.y:166
 		{
 			yyVAL.val = "string"
 		}
-	case 35:
+	case 36:
-		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:159
-		{
-			yyVAL.val = "byte"
-		}
-	case 40:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line sunrpc.y:167
 		{
-			yyVAL.val = "int64"
+			yyVAL.val = "byte"
 		}
 	case 41:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:168
+		//line sunrpc.y:175
 		{
-			yyVAL.val = "int32"
+			yyVAL.val = "int64"
 		}
 	case 42:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:169
+		//line sunrpc.y:176
 		{
-			yyVAL.val = "int16"
+			yyVAL.val = "int32"
 		}
 	case 43:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:170
+		//line sunrpc.y:177
+		{
+			yyVAL.val = "int16"
+		}
+	case 44:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sunrpc.y:178
 		{
 			yyVAL.val = "int8"
 		}
-	case 45:
+	case 46:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line sunrpc.y:178
+		//line sunrpc.y:186
 		{
 			AddFixedArray(yyDollar[2].val, yyDollar[1].val, yyDollar[4].val)
 		}
-	case 46:
+	case 47:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line sunrpc.y:182
+		//line sunrpc.y:190
 		{
 			AddVariableArray(yyDollar[2].val, yyDollar[1].val, yyDollar[4].val)
 		}
-	case 47:
+	case 48:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line sunrpc.y:183
+		//line sunrpc.y:191
 		{
 			AddVariableArray(yyDollar[2].val, yyDollar[1].val, "")
 		}
-	case 48:
+	case 49:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line sunrpc.y:191
+		//line sunrpc.y:199
 		{
 			AddOptValue(yyDollar[3].val, yyDollar[1].val)
 		}
-	case 49:
+	case 50:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line sunrpc.y:195
+		//line sunrpc.y:203
 		{
 			StartStruct(yyDollar[2].val)
 		}
-	case 50:
+	case 51:
 		yyDollar = yyS[yypt-6 : yypt+1]
-		//line sunrpc.y:195
+		//line sunrpc.y:203
 		{
 			AddStruct()
 		}
-	case 54:
+	case 55:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line sunrpc.y:208
+		//line sunrpc.y:216
 		{
 			StartUnion(yyDollar[2].val)
 		}
-	case 55:
+	case 56:
 		yyDollar = yyS[yypt-10 : yypt+1]
-		//line sunrpc.y:208
+		//line sunrpc.y:216
 		{
 			AddUnion()
 		}
-	case 59:
+	case 60:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line sunrpc.y:221
+		//line sunrpc.y:229
 		{
 			StartCase(yyDollar[2].val)
 		}
-	case 60:
+	case 61:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line sunrpc.y:221
+		//line sunrpc.y:229
 		{
 			AddCase()
 		}
-	case 61:
+	case 62:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line sunrpc.y:222
+		//line sunrpc.y:230
 		{
 			StartCase("default")
 		}
-	case 62:
+	case 63:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line sunrpc.y:222
+		//line sunrpc.y:230
 		{
 			AddCase()
 		}

libvirt_integration_test.go

@@ -17,6 +17,7 @@
 package libvirt
 
 import (
+	"bytes"
 	"encoding/xml"
 	"net"
 	"sync"
@@ -252,6 +253,58 @@ func TestXMLIntegration(t *testing.T) {
 	}
 }
 
+func TestVolumeUploadDownloadIntegration(t *testing.T) {
+	testdata := []byte("Hello, world!")
+	l := New(testConn(t))
+
+	if err := l.Connect(); err != nil {
+		t.Error(err)
+	}
+	defer l.Disconnect()
+
+	pool, err := l.StoragePool("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var volObj struct {
+		XMLName  xml.Name `xml:"volume"`
+		Name     string   `xml:"name"`
+		Capacity struct {
+			Value uint64 `xml:",chardata"`
+		} `xml:"capacity"`
+		Target struct {
+			Format struct {
+				Type string `xml:"type,attr"`
+			} `xml:"format"`
+		} `xml:"target"`
+	}
+	volObj.Name = "testvol"
+	volObj.Capacity.Value = uint64(len(testdata))
+	volObj.Target.Format.Type = "raw"
+	xmlVol, err := xml.Marshal(volObj)
+	if err != nil {
+		t.Fatal(err)
+	}
+	vol, err := l.StorageVolCreateXML(pool, string(xmlVol), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.StorageVolDelete(vol, 0)
+	err = l.StorageVolUpload(vol, bytes.NewBuffer(testdata), 0, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	err = l.StorageVolDownload(vol, &buf, 0, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(testdata, buf.Bytes()) != 0 {
+		t.Fatal("download not what we uploaded")
+	}
+}
+
 // verify we're able to concurrently communicate with libvirtd.
 // see: https://github.com/digitalocean/go-libvirt/pull/52
 func Test_concurrentWrite(t *testing.T) {

rpc.go

@@ -174,10 +174,12 @@ func (l *Libvirt) callback(id uint32, res response) {
 	c, ok := l.callbacks[id]
 	l.cm.Unlock()
 	if ok {
+		// we close channel in deregister() so that we don't block here forever without receiver
+		defer func() {
+			recover()
+		}()
 		c <- res
 	}
-
-	l.deregister(id)
 }
 
 // route sends incoming packets to their listeners.
@@ -274,11 +276,107 @@ func (l *Libvirt) deregister(id uint32) {
 // returns response returned by server.
 // if response is not OK, decodes error from it and returns it.
 func (l *Libvirt) request(proc uint32, program uint32, payload []byte) (response, error) {
+	return l.requestStream(proc, program, payload, nil, nil)
+}
+
+func (l *Libvirt) processIncomingStream(c chan response, inStream io.Writer) (response, error) {
+	for {
+		resp, err := l.getResponse(c)
+		if err != nil {
+			return resp, err
+		}
+		// StatusOK here means end of stream
+		if resp.Status == StatusOK {
+			return resp, nil
+		}
+		// StatusError is handled in getResponse, so this is StatusContinue
+		// StatusContinue is valid here only for stream packets
+		// libvirtd breaks protocol and returns StatusContinue with empty Payload when stream finishes
+		if len(resp.Payload) == 0 {
+			return resp, nil
+		}
+		if inStream != nil {
+			_, err = inStream.Write(resp.Payload)
+			if err != nil {
+				return response{}, err
+			}
+		}
+	}
+}
+
+func (l *Libvirt) requestStream(proc uint32, program uint32, payload []byte, outStream io.Reader, inStream io.Writer) (response, error) {
 	serial := l.serial()
 	c := make(chan response)
 
 	l.register(serial, c)
+	defer l.deregister(serial)
 
+	err := l.sendPacket(serial, proc, program, payload, Call, StatusOK)
+	if err != nil {
+		return response{}, err
+	}
+
+	resp, err := l.getResponse(c)
+	if err != nil {
+		return resp, err
+	}
+
+	if outStream != nil {
+		abortOutStream := make(chan bool)
+		outStreamErr := make(chan error)
+		go func() {
+			outStreamErr <- l.sendStream(serial, proc, program, outStream, abortOutStream)
+		}()
+
+		// Even without incoming stream server sends confirmation once all data is received
+		resp, err = l.processIncomingStream(c, inStream)
+		if err != nil {
+			abortOutStream <- true
+			return resp, err
+		}
+
+		err = <-outStreamErr
+		if err != nil {
+			return response{}, err
+		}
+	} else if inStream != nil {
+		return l.processIncomingStream(c, inStream)
+	}
+
+	return resp, nil
+}
+
+func (l *Libvirt) sendStream(serial uint32, proc uint32, program uint32, stream io.Reader, abort chan bool) error {
+	// Keep total packet length under 4 MiB to follow possible limitation in libvirt server code
+	buf := make([]byte, 4*MiB-constants.HeaderSize)
+	for {
+		select {
+		case <-abort:
+			return l.sendPacket(serial, proc, program, nil, Stream, StatusError)
+		default:
+		}
+		n, err := stream.Read(buf)
+		if err != nil {
+			if err == io.EOF {
+				return l.sendPacket(serial, proc, program, nil, Stream, StatusOK)
+			}
+			// keep original error
+			err2 := l.sendPacket(serial, proc, program, nil, Stream, StatusError)
+			if err2 != nil {
+				return err2
+			}
+			return err
+		}
+		if n > 0 {
+			err = l.sendPacket(serial, proc, program, buf[:n], Stream, StatusContinue)
+			if err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func (l *Libvirt) sendPacket(serial uint32, proc uint32, program uint32, payload []byte, typ uint32, status uint32) error {
 	size := constants.PacketLengthSize + constants.HeaderSize
 	if payload != nil {
 		size += len(payload)
@@ -290,9 +388,9 @@ func (l *Libvirt) request(proc uint32, program uint32, payload []byte) (response
 			Program:   program,
 			Version:   constants.ProtocolVersion,
 			Procedure: proc,
-			Type:      Call,
+			Type:      typ,
 			Serial:    serial,
-			Status:    StatusOK,
+			Status:    status,
 		},
 	}
 
@@ -301,23 +399,23 @@ func (l *Libvirt) request(proc uint32, program uint32, payload []byte) (response
 	defer l.mu.Unlock()
 	err := binary.Write(l.w, binary.BigEndian, p)
 	if err != nil {
-		return response{}, err
+		return err
 	}
 
 	// write payload
 	if payload != nil {
 		err = binary.Write(l.w, binary.BigEndian, payload)
 		if err != nil {
-			return response{}, err
+			return err
 		}
 	}
 
-	if err := l.w.Flush(); err != nil {
+	return l.w.Flush()
-		return response{}, err
+}
-	}
 
+func (l *Libvirt) getResponse(c chan response) (response, error) {
 	resp := <-c
-	if resp.Status != StatusOK {
+	if resp.Status == StatusError {
 		return resp, decodeError(resp.Payload)
 	}
 

units.go

@@ -0,0 +1,27 @@
+// Copyright 2016 The go-libvirt Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This module provides different units of measurement to make other
+// code more readable.
+
+package libvirt
+
+const (
+    // B - byte
+	B   = 1
+    // KiB - kibibyte
+	KiB = 1024 * B
+    // MiB - mebibyte
+	MiB = 1024 * KiB
+)