commit f29845d3bba7032c7f61d25a34441d9ab7ff8a4e
parent 4125cc7cee7817ec5c7e2e0328011f3e234a4b46
Author: Mattias Andrée <maandree@kth.se>
Date: Sat, 14 Jan 2017 06:33:00 +0100
Add definition checks for posix_fadvise flags + blind-concat: add -j
Signed-off-by: Mattias Andrée <maandree@kth.se>
Diffstat:
8 files changed, 166 insertions(+), 13 deletions(-)
diff --git a/TODO b/TODO
@@ -11,7 +11,6 @@ blind-apply-kernel apply a convolution matrix
UNTESTED:
blind-arithm
blind-colour-srgb
- blind-concat
blind-crop
blind-cut
blind-dissolve
@@ -27,5 +26,4 @@ UNTESTED:
blind-set-saturation
blind-single-colour
blind-split
- blind-stack
blind-to-text
diff --git a/config.mk b/config.mk
@@ -1,5 +1,5 @@
-# You may want to remove -DHAVE_PRCTL from CPPFLAGS if you are not using Linux.
+# You may want to remove -DHAVE_PRCTL and -DHAVE_EPOLL from CPPFLAGS if you are not using Linux.
CFLAGS = -std=c99 -Wall -pedantic -O2
-CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 -D_FILE_OFFSET_BITS=64 -DHAVE_PRCTL
+CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 -D_FILE_OFFSET_BITS=64 -DHAVE_PRCTL -DHAVE_EPOLL
LDFLAGS = -lm -s
diff --git a/src/blind-concat.c b/src/blind-concat.c
@@ -2,16 +2,21 @@
#include "stream.h"
#include "util.h"
+#if defined(HAVE_SYS_EPOLL_H)
+# include <sys/epoll.h>
+#endif
#include <sys/mman.h>
+#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
+#include <limits.h>
#include <string.h>
#include <unistd.h>
-USAGE("[-o output-file] first-stream ... last-stream")
+USAGE("[-o output-file [-j jobs]] first-stream ... last-stream")
static void
-concat_to_stdout(int argc, char *argv[])
+concat_to_stdout(int argc, char *argv[], const char *fname)
{
struct stream *streams;
size_t frames = 0;
@@ -32,11 +37,11 @@ concat_to_stdout(int argc, char *argv[])
streams->frames = frames;
fprint_stream_head(stdout, streams);
- efflush(stdout, "<stdout>");
+ efflush(stdout, fname);
for (i = 0; i < argc; i++) {
for (; eread_stream(streams + i, SIZE_MAX); streams[i].ptr = 0)
- ewriteall(STDOUT_FILENO, streams[i].buf, streams[i].ptr, "<stdout>");
+ ewriteall(STDOUT_FILENO, streams[i].buf, streams[i].ptr, fname);
close(streams[i].fd);
}
@@ -90,26 +95,144 @@ concat_to_file(int argc, char *argv[], char *output_file)
close(fd);
}
+static void
+concat_to_file_parallel(int argc, char *argv[], char *output_file, size_t jobs)
+{
+ struct epoll_event *events;
+ struct stream *streams;
+ size_t *ptrs;
+ char head[STREAM_HEAD_MAX];
+ size_t ptr, frame_size, frames = 0, next = 0, j;
+ ssize_t headlen;
+ int fd, i, n, pollfd;
+
+#if !defined(HAVE_SYS_EPOLL_H)
+ fd = eopen(output_file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
+ if (fd != STDOUT_FILENO && dup2(fd, STDOUT_FILENO) == -1)
+ eprintf("dup2:");
+ concat_to_stdout(argc, argv, output_file);
+ return;
+#else
+
+ if (jobs > (size_t)argc)
+ jobs = (size_t)argc;
+
+ fd = eopen(output_file, O_RDWR | O_CREAT | O_TRUNC, 0666);
+ events = emalloc(jobs * sizeof(*events));
+ streams = emalloc((size_t)argc * sizeof(*streams));
+ ptrs = emalloc((size_t)argc * sizeof(*ptrs));
+
+ for (i = 0; i < argc; i++) {
+ streams[i].file = argv[i];
+ streams[i].fd = eopen(streams[i].file, O_RDONLY);
+ einit_stream(streams + i);
+ if (i)
+ echeck_compat(streams + i, streams);
+ if (streams[i].frames > SIZE_MAX - frames)
+ eprintf("resulting video is too long\n");
+ frames += streams[i].frames;
+ }
+
+ sprintf(head, "%zu %zu %zu %s\n%cuivf%zn",
+ frames, streams->width, streams->height, streams->pixfmt, 0, &headlen);
+
+ echeck_frame_size(streams->width, streams->height, streams->pixel_size, 0, output_file);
+ frame_size = streams->width * streams->height * streams->pixel_size;
+ ptr = (size_t)headlen;
+ for (i = 0; i < argc; i++) {
+ ptrs[i] = ptr;
+ ptr += streams->frames * frame_size;
+ }
+ if (ftruncate(fd, (off_t)ptr))
+ eprintf("ftruncate %s:", output_file);
+#if defined(POSIX_FADV_RANDOM)
+ posix_fadvise(fd, (size_t)headlen, 0, POSIX_FADV_RANDOM);
+#endif
+
+ pollfd = epoll_create1(0);
+ if (pollfd == -1)
+ eprintf("epoll_create1:");
+
+ epwriteall(fd, head, (size_t)head_len, 0, output_file);
+ for (i = 0; i < argc; i++) {
+ epwriteall(fd, streams[i].buf, streams[i].ptr, ptrs[i], output_file);
+ ptrs[i] += streams[i].ptr;
+ streams[i].ptr = 0;
+ }
+
+ for (j = 0; j < jobs; j++, next++) {
+ events->events = EPOLLIN;
+ events->data.u64 = next;
+ if (epoll_ctl(pollfd, EPOLL_CTL_ADD, streams[next].fd, events) == -1) {
+ if ((errno == ENOMEM || errno == ENOSPC) && j)
+ break;
+ eprintf("epoll_ctl:");
+ }
+ }
+ jobs = j;
+
+ while (jobs) {
+ n = epoll_wait(pollfd, events, jobs, -1);
+ if (n < 0)
+ eprintf("epoll_wait:");
+ for (i = 0; i < n; i++) {
+ j = events[i].data.u64;
+ if (!eread_stream(streams + j)) {
+ close(streams[j].fd);
+ if (next < (size_t)argc) {
+ events->events = EPOLLIN;
+ events->data.u64 = next;
+ if (epoll_ctl(pollfd, EPOLL_CTL_ADD, streams[next].fd, events) == -1) {
+ if ((errno == ENOMEM || errno == ENOSPC) && j)
+ break;
+ eprintf("epoll_ctl:");
+ }
+ next++;
+ } else {
+ jobs--;
+ }
+ } else {
+ epwriteall(fd, streams[j].buf, streams[j].ptr, ptrs[j], output_file);
+ ptrs[j] += streams[j].ptr;
+ streams[j].ptr = 0;
+ }
+ }
+ }
+
+ close(pollfd);
+ free(events);
+ free(streams);
+ free(ptrs);
+
+#endif
+}
+
int
main(int argc, char *argv[])
{
char *output_file = NULL;
+ size_t jobs = 0;
ARGBEGIN {
case 'o':
output_file = EARG();
break;
+ case 'j':
+ jobs = etozu_flag('j', EARG(), 1, SHRT_MAX);
+ break;
default:
usage();
} ARGEND;
- if (argc < 2)
+ if (argc < 2 || (jobs && !output_file))
usage();
- if (output_file)
+ if (jobs)
+ concat_to_file_parallel(argc, argv, output_file, jobs);
+ else if (output_file)
concat_to_file(argc, argv, output_file);
else
- concat_to_stdout(argc, argv);
+ concat_to_stdout(argc, argv, "<stdout>");
return 0;
}
diff --git a/src/blind-cut.c b/src/blind-cut.c
@@ -53,7 +53,9 @@ main(int argc, char *argv[])
end = end * frame_size + stream.headlen;
start = start * frame_size + stream.headlen;
+#if defined(POSIX_FADV_SEQUENTIAL)
posix_fadvise(stream.fd, start, end - start, POSIX_FADV_SEQUENTIAL);
+#endif
for (ptr = start; ptr < end; ptr += (size_t)r) {
max = end - ptr;
max = max < sizeof(buf) ? max : sizeof(buf);
diff --git a/src/blind-repeat.c b/src/blind-repeat.c
@@ -39,7 +39,9 @@ main(int argc, char *argv[])
efflush(stdout, "<stdout>");
while (inf || count--) {
+#if defined(POSIX_FADV_SEQUENTIAL)
posix_fadvise(stream.fd, stream.headlen, 0, POSIX_FADV_SEQUENTIAL);
+#endif
for (ptr = stream.headlen;; ptr += (size_t)r) {
r = pread(stream.fd, buf, sizeof(buf), ptr);
if (r < 0)
diff --git a/src/blind-reverse.c b/src/blind-reverse.c
@@ -27,7 +27,9 @@ main(int argc, char *argv[])
if (stream.frames > SSIZE_MAX / frame_size)
eprintf("%s: video is too large\n", stream.file);
+#if defined(POSIX_FADV_RANDOM)
posix_fadvise(stream.fd, 0, 0, POSIX_FADV_RANDOM);
+#endif
while (stream.frames--) {
ptr = stream.frames * frame_size;
end = ptr + frame_size;
diff --git a/src/util.c b/src/util.c
@@ -140,6 +140,22 @@ readall(int fd, void *buf, size_t n)
return r;
}
+int
+pwriteall(int fd, void *buf, size_t n, size_t ptr)
+{
+ char *buffer = buf;
+ ssize_t r;
+ while (n) {
+ r = pwrite(fd, buffer, n, ptr);
+ if (r < 0)
+ return -1;
+ buffer += (size_t)r;
+ n -= (size_t)r;
+ ptr += (size_t)r;
+ }
+ return 0;
+}
+
static inline pid_t
enfork(int status)
diff --git a/src/util/io.h b/src/util/io.h
@@ -1,7 +1,8 @@
/* See LICENSE file for copyright and license details. */
-#define ewriteall(...) enwriteall(1, __VA_ARGS__)
-#define ereadall(...) enreadall(1, __VA_ARGS__)
+#define ewriteall(...) enwriteall(1, __VA_ARGS__)
+#define ereadall(...) enreadall(1, __VA_ARGS__)
+#define epwriteall(...) enpwriteall(1, __VA_ARGS__)
int writeall(int fd, void *buf, size_t n);
@@ -22,3 +23,12 @@ enreadall(int status, int fd, void *buf, size_t n, const char *fname)
enprintf(status, "read %s:", fname);
return (size_t)r;
}
+
+int pwriteall(int fd, void *buf, size_t n, size_t ptr);
+
+static inline void
+enpwriteall(int status, int fd, void *buf, size_t n, size_t ptr, const char *fname)
+{
+ if (pwriteall(fd, buf, n, ptr))
+ enprintf(status, "pwrite %s:", fname);
+}
