Gulp-filter in one stream not delivering all files for dest

Question

In gulp I’m currently trying to batch copy, move and apply plugins to a bunch of src files for a build process.Trying to use gulp-filter to split the required files up for each step and applying the correct plugins to each, it appears to work but only for some of the files.

The problem I’m getting is the final output in “dist” is only partially showing the files loaded. I.E. it’s only showing css, and img folders and only half the images, likewise only half of the lib's all pulled across. Fairly sure it isn’t one of the plugins silently failing. Also sometimes it wasn't going fully recursive on src files.

Tried removing step steps like the image plugin but still had same issues.

This is ideally what I want to use (and am currently using but has those issues):

var sourceFiles = [
    '!./js/_src',
    './js/**',
    './css/**/*.css',
    './img/**',
    './files/**',
    '!./login/_src',
    './login/**'
]
gulp.task('build',['clean'], function(){

    var replaceVersion = [/%version/gi,'v=' + p.version ];

    var filterHTML  = filter('**/*.html');
    var filterJSON  = filter('**/*.json');
    var filterENV   = filter('**/env.js');
    var filterIMG   = filter('img/**/*.*');

    gulp
        .src(sourceFiles, {base: './'})

        .pipe(plumber({
            errorHandler: onError
        }))

        .pipe(filterHTML)
            .pipe(replace([replaceVersion]))
            .pipe(minifyhtml())
        .pipe(filterHTML.restore())

        .pipe(filterJSON)
            .pipe(minifyjson())
        .pipe(filterJSON.restore())

        .pipe(filterENV)
            .pipe(replace([replaceVersion]))
        .pipe(filterENV.restore())

        .pipe(filterIMG)
            .pipe(imagemin({
                progressive: true,
                svgoPlugins: [{removeViewBox: false}],
                use: [pngcrush()]
            }))
        .pipe(filterIMG.restore())

        .pipe(gulp.dest(paths.dist))

        .pipe(notify({
            title: 'Gulp BUILD Success ',
            icon: path.join(__dirname, '/_assets/gulp.png'),
            onLast: true,
            wait: true
        }));

}); 

And this is a dirty/messy/long-winded way I'm currently settling for until I find a solution :

gulp.task('build', function(){

    console.log('----------------------------------------------------');
    console.log('                     v'+ p.version);
    console.log('----------------------------------------------------');

    setTimeout(function(){ // few seconds delay to display message above

        runSequence(
            [
                'clean'
            ],
            [
                'build:index',
                'build:styles',
                'build:files',
                'build:icons',
                'build:scripts',
                'build:libs',
                'build:templates',
                'build:resource',
                'build:img',
                'build:login',
                'build:signup',
                'build:recommend'
            ],
            [
                'build:setVersion'
            ],
            [
                'build:compressHTML',
                'build:compressJSON',
                'build:compressIMG'
            ],
            function(){
                console.log('Build Complete');
            }
        )

    }, 3000)

});

gulp.task('build:index', function(){
    return gulp
        .src(['index.html'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:styles', function(){
    return gulp
        .src(['css/**/*.css', 'css/maps/*.*'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:files', function(){
    return gulp
        .src('files/**/*.*', { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:icons', function(){
    return gulp
        .src('icons/**/*.*', { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:scripts', function(){
    return gulp
        .src([‘js/app.min.js', 'js/env.js', 'js/main.js', 'js/maps/app.min.js.map'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:libs', function(){
    return gulp
        .src(['js/lib/**/*.*','!js/lib/_src/**'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:templates', function(){
    gulp
        .src(['js/templates/**/*.*'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:resource', function(){
    return gulp
        .src(['js/resource/*.*'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:img', function(){
    return gulp
        .src(['img/**/*.*'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:login', function(){
    return gulp
        .src(['login/**/*.*' ,'!login/_src/**'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:signup', function(){
    return gulp
        .src(['signup/**/*.*' ,'!signup/_src/**'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});

gulp.task('build:recommend', function(){
    return gulp
        .src(['recommend/**/*.*' ,'!recommend/_src/**'], { base: './'})
        .pipe(gulp.dest(paths.dist));
});


gulp.task('build:setVersion', function(){
    return gulp
        .src([paths.dist + 'index.html', paths.dist + '/**/*.html', paths.dist + '/**/env.js'], { base: './'})
        .pipe(replace([version]))
        .pipe(gulp.dest('.'));
});


gulp.task('build:compressHTML', function(){
    return gulp
        .src([paths.dist + '/**/*.html'], { base: './'})
        .pipe(minifyhtml())
        .pipe(gulp.dest('.'));
});

gulp.task('build:compressJSON', function(){
    return gulp
        .src([paths.dist + '/**/*.json'], { base: './'})
        .pipe(minifyjson())
        .pipe(gulp.dest('.'));
});

gulp.task('build:compressIMG', function(){
    return gulp
        .src([paths.dist + '/img/**/*.*'], { base: './'})
        .pipe(imagemin({
            progressive: true,
            svgoPlugins: [{removeViewBox: false}],
            use: [pngcrush()]
        }))
        .pipe(gulp.dest('.'));
});

Answer 1

If you want one task that does everything, I would recommend a different approach. Instead of srcing all of your files and then filtering, create multiple streams and then merge them together.

var merge = require('merge-stream');
gulp.task('build',['clean'], function(){

    var replaceVersion = [/%version/gi,'v=' + p.version ];

    var htmlStream = gulp.src('**/*.html', base: './'})
            .pipe(replace([replaceVersion]))
            .pipe(minifyhtml()),
        jsonStream = gulp.src('**/*.json', base: './'})
            .pipe(minifyjson())
        envStream = gulp.src('**/env.js', base: './'})
            .pipe(replace([replaceVersion]))
        imgStream = gulp.src('img/**/*.*', base: './'})
            .pipe(imagemin({
                progressive: true,
                svgoPlugins: [{removeViewBox: false}],
                use: [pngcrush()]
            }));

    return merge([htmlStream, jsonStream, envStream, imgStream])
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(filterIMG.restore());
});

I have had success with this. With each stream being specified separately, it is more readable to me.

Answer 2

This is a bug, we fixed it in the last major version. https://github.com/sindresorhus/gulp-filter/issues/29