From 779888af6bc05f2395e0c9fc0c53067e0674cb27 Mon Sep 17 00:00:00 2001 From: John Tsiombikas Date: Thu, 30 Jul 2020 14:22:42 +0300 Subject: [PATCH] reposerve server half way done --- protocol | 76 +++++++++++++++++++++++++++ server/Makefile | 4 +- server/src/client.c | 135 ++++++++++++++++++++++++++++++++++++++++++++++++ server/src/client.h | 22 ++++++++ server/src/main.c | 9 +++- src/proto.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/proto.h | 34 +++++++++++++ 7 files changed, 417 insertions(+), 4 deletions(-) create mode 100644 protocol create mode 100644 server/src/client.c create mode 100644 server/src/client.h create mode 100644 src/proto.c create mode 100644 src/proto.h diff --git a/protocol b/protocol new file mode 100644 index 0000000..4839b94 --- /dev/null +++ b/protocol @@ -0,0 +1,76 @@ +Reposerve protocol v1 +===================== + +Commands are variable-length strings ending with a newline character (10), with +a maximum length of 256 characters, including the newline. + +Responses always start with a result line with either "OK", followed by a byte +count of response data to follow, or an "ERR" followed by silence. + +All binary values in response data are in little-endian byte order (LSB-first). + +On connection establishment, the server sends a string of the form: + - protocol: + +DOS filename mode +----------------- +Request: "dos" +Description: +When the client sends a dos request, the server enables DOS filename +translation. Filename path components are automatically converted to DOS 8.3 +form when transmitted to the client. File lists transmitted by the client are +translated back to the original names, capital letters for new untracked files +are converted to lowercase, except for specific files such as "Makefile*", +"GNUmakefile", "README*", and "COPYING*", otherwise for tracked files, the +existing capitalization is maintained. + +File list +--------- +Request: "flist" +Response data: + + offset | size | description + 0 | 4 | number of file entries (N) + 4 | 32 | file entry 0 + 4+32 | 32 | file entry 1 + ... + 4+i*32 | 32 | file entry i + ... + 4+N*32 | ?? | file name table + + + File entry (32 bytes): + offset | size | description + 0 | 16 | md5 checksum + 16 | 4 | last modification timestamp + 20 | 4 | file size + 24 | 4 | name string offset (from the start of the name table) + 28 | 4 | name string length + +Description: +Provides a list of files managed by the repo server, with all the relevant +metadata about each file. + +Push files +---------- +Request: "push" +Request data: Exactly the same as the "flist" response data, except that after +each file name in the name table, the contents of the corresponding file are +transmitted. + +Description: +Although potentially every file can be sent using this request, the intent is to +only push modified files according to either the modification timestamp or the +checksum provided in the file list. + +Pull files +---------- +Request: "pull" +Request data: Exactly the same as the "flist" response data. +Response data: Exactly the same as the "flist" response data, except that after +the file name in the name table, the contents of the corresponding file are +transmitted. + +Description: +The server compares file checksums or modification timestamps in the request +file list, and sends back only those files which are modified. diff --git a/server/Makefile b/server/Makefile index 97588d4..3a2ad08 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,11 +1,11 @@ PREFIX ?= /usr/local -src = $(wildcard src/*.c) +src = $(wildcard src/*.c) $(wildcard ../src/*.c) obj = $(src:.c=.o) dep = $(src:.c=.d) bin = reposerve -CFLAGS = -pedantic -Wall -g -MMD +CFLAGS = -pedantic -Wall -g -MMD -Isrc -I../src LDFLAGS = -lgit2 $(bin): $(obj) diff --git a/server/src/client.c b/server/src/client.c new file mode 100644 index 0000000..bec2f11 --- /dev/null +++ b/server/src/client.c @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include "client.h" +#include "proto.h" +#include "repo.h" + +static int proc_request(struct client *c); +static int proc_flist(struct client *c, char *data, int datasz); +static int proc_push(struct client *c, char *data, int datasz); +static int proc_pull(struct client *c, char *data, int datasz); + +const char *repo_path; + +int handle_client(struct client *c) +{ + char buf[512]; + int sz, left; + + while((sz = read(c->s, buf, sizeof buf)) > 0) { + left = sizeof c->buf - c->bsz; + if(sz > left) sz = left; + memcpy(c->buf + c->bsz, buf, sz); + c->bsz += sz; + } + + if(proc_request(c) == -1) { + return -1; + } + return 0; +} + +static int proc_request(struct client *c) +{ + char *endp = c->buf; + char *reqdata; + int reqdata_len; + + while(endp < c->buf + c->bsz) { + if(*endp == '\n') break; + } + if(endp >= c->buf + c->bsz) { + return 0; + } + *endp = 0; + + reqdata = endp + 1; + reqdata_len = c->bsz - (reqdata - c->buf); + + c->bsz = 0; + + if(strcmp(c->buf, "flist") == 0) { + return proc_flist(c, reqdata, reqdata_len); + } else if(strcmp(c->buf, "push") == 0) { + return proc_push(c, reqdata, reqdata_len); + } else if(strcmp(c->buf, "pull") == 0) { + return proc_pull(c, reqdata, reqdata_len); + } else if(strcmp(c->buf, "dos") == 0) { + c->flags |= CLIENT_DOS; + send_ok(c, 0); + return 0; + } + + send_err(c); + return 0; +} + +void send_string(struct client *c, const char *s) +{ + write(c->s, s, strlen(s)); +} + +void send_ok(struct client *c, int resp_sz) +{ + char buf[32]; + sprintf(buf, "OK %d\n", resp_sz < 0 ? 0 : resp_sz); + send_string(c, buf); +} + +void send_err(struct client *c) +{ + send_string(c, "ERR\n"); +} + +static struct flist *gen_flist(int contents) +{ + int i, count; + struct flist *flist; + + if(!(flist = flist_create())) { + return 0; + } + + if(repo_init(repo_path) == -1) { + return 0; + } + count = repo_num_files(); + + for(i=0; ipath, contents); + } + repo_cleanup(); + + return flist; +} + +static int proc_flist(struct client *c, char *data, int datasz) +{ + struct flist *flist; + + if(!(flist = gen_flist(0))) { + send_err(c); + return -1; + } + flist_finalize(flist); + + send_ok(c, flist->final_size); + write(c->s, flist->flist, flist->final_size); + + flist_destroy(flist); + return 0; +} + +static int proc_push(struct client *c, char *data, int datasz) +{ + send_err(c); + return 0; +} + +static int proc_pull(struct client *c, char *data, int datasz) +{ + send_err(c); + return 0; +} diff --git a/server/src/client.h b/server/src/client.h new file mode 100644 index 0000000..10720db --- /dev/null +++ b/server/src/client.h @@ -0,0 +1,22 @@ +#ifndef CLIENT_H_ +#define CLIENT_H_ + +enum { + CLIENT_DOS = 1 +}; + +struct client { + int s; + unsigned int flags; + char buf[16384]; + int bsz; + struct client *next; +}; + +int handle_client(struct client *c); + +void send_string(struct client *c, const char *s); +void send_ok(struct client *c, int resp_sz); +void send_err(struct client *c); + +#endif /* CLIENT_H_ */ diff --git a/server/src/main.c b/server/src/main.c index 12bb01f..ab5c055 100644 --- a/server/src/main.c +++ b/server/src/main.c @@ -15,12 +15,15 @@ static const char *guess_repo_name(const char *path); static int parse_args(int argc, char **argv); -static const char *repo_path, *repo_name; +static const char *repo_name; static int lis = -1; static int port = 64357; static struct client *clients; +const char *repo_path; + + int main(int argc, char **argv) { struct sockaddr_in addr; @@ -101,7 +104,7 @@ int main(int argc, char **argv) perror("failed to accept connection"); continue; } - if(!(c = malloc(sizeof *c))) { + if(!(c = calloc(1, sizeof *c))) { perror("failed to allocate memory for client structure"); close(s); continue; @@ -109,6 +112,8 @@ int main(int argc, char **argv) c->s = s; c->next = clients; clients = c; + + send_string(c, "reposerve-0.1 protocol:0\n"); } } diff --git a/src/proto.c b/src/proto.c new file mode 100644 index 0000000..9d548a6 --- /dev/null +++ b/src/proto.c @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include "proto.h" +#include "md5.h" + +struct flist *flist_create(void) +{ + struct flist *flist; + + if(!(flist = calloc(1, sizeof *flist))) { + perror("flist_create"); + return 0; + } + flist->final_size = -1; + return flist; +} + +void flist_destroy(struct flist *flist) +{ + if(flist) { + free(flist->flist); + free(flist->data); + free(flist); + } +} + +int flist_add(struct flist *flist, const char *fname, int contents) +{ + FILE *fp; + struct proto_file_entry fent; + int namelen, datalen, sz; + struct stat st; + char *dptr; + char buf[1024]; + struct md5_state md; + + if(flist->final_size >= 0) { + fprintf(stderr, "flist_add: can't add more files to a finalized file list\n"); + return -1; + } + + if(!(fp = fopen(fname, "rb"))) { + fprintf(stderr, "flist_add: failed to open file: %s: %s\n", fname, strerror(errno)); + return -1; + } + fstat(fileno(fp), &st); + + fent.size = st.st_size; + fent.mtime = st.st_mtime; + + namelen = datalen = strlen(fname); + if(contents) { + datalen += fent.size; + } + + if(flist->flist->num_files >= flist->max_files) { + struct proto_flist *tmp; + int newsz = flist->max_files ? flist->max_files << 1 : 32; + + if(!(tmp = realloc(flist->flist, sizeof *tmp + (newsz - 1) * sizeof *tmp->files))) { + fprintf(stderr, "flist_add: failed to resize file list to %d entries\n", newsz); + return -1; + } + flist->flist = tmp; + flist->max_files = newsz; + } + if(flist->data_sz + datalen >= flist->max_data_sz) { + char *tmp; + int newsz = flist->max_data_sz + datalen; + newsz |= newsz >> 1; + newsz |= newsz >> 2; + newsz |= newsz >> 4; + newsz |= newsz >> 8; + newsz |= newsz >> 16; + newsz += 1; + + if(!(tmp = realloc(flist->data, newsz))) { + fprintf(stderr, "flist_add: failed to resize data buffer to %d bytes\n", newsz); + return -1; + } + flist->data = tmp; + flist->max_data_sz = newsz; + } + + fent.nameoffs = flist->data_sz; + fent.namelen = namelen; + dptr = flist->data + flist->data_sz; + memcpy(dptr, fname, namelen); + + md5_begin(&md); + if(contents) { + dptr += namelen; + + while((sz = fread(dptr, 1, st.st_size, fp)) > 0) { + md5_msg(&md, dptr, sz); + dptr += sz; + st.st_size -= sz; + } + } else { + while((sz = fread(buf, 1, sizeof buf, fp)) > 0) { + md5_msg(&md, buf, sz); + } + } + md5_end(&md); + memcpy(fent.csum, md.sum, sizeof fent.csum); + + fclose(fp); + + flist->flist->files[flist->flist->num_files++] = fent; + flist->data_sz += datalen; + return 0; +} + +int flist_finalize(struct flist *flist) +{ + int newsz; + struct proto_flist *tmp; + + if(!flist->flist || flist->flist->num_files <= 0) { + fprintf(stderr, "flist_finalize: nothing to finalize, file list empty\n"); + return -1; + } + + newsz = sizeof *tmp + (flist->flist->num_files - 1) * sizeof *tmp->files + flist->data_sz; + if(!(tmp = realloc(flist->flist, newsz))) { + fprintf(stderr, "flist_finalize: failed to resize file list buffer to %d bytes\n", newsz); + return -1; + } + + memcpy(tmp->files + flist->flist->num_files, flist->data, flist->data_sz); + free(flist->data); + + memset(flist, 0, sizeof *flist); + flist->flist = tmp; + flist->final_size = newsz; + return 0; +} diff --git a/src/proto.h b/src/proto.h new file mode 100644 index 0000000..e7a043a --- /dev/null +++ b/src/proto.h @@ -0,0 +1,34 @@ +#ifndef PROTO_H_ +#define PROTO_H_ + +#include + +struct proto_file_entry { + uint32_t csum[4]; + uint32_t mtime; + uint32_t size; + uint32_t nameoffs; + uint32_t namelen; +}; + +struct proto_flist { + uint32_t num_files; + struct proto_file_entry files[1]; +}; + +struct flist { + struct proto_flist *flist; + int max_files; + char *data; + int data_sz, max_data_sz; + + int final_size; +}; + +struct flist *flist_create(void); +void flist_destroy(struct flist *flist); + +int flist_add(struct flist *flist, const char *fname, int contents); +int flist_finalize(struct flist *flist); + +#endif /* PROTO_H_ */ -- 1.7.10.4