Drupal 8: Move Deprecated drupal/drupal Composer Template to drupal-composer/drupal-project

Submitted by nigel on Wednesday 11th December 2019

Tutorial for moving an existing Drupal 8 codebase away from the deprecated drupal/drupal composer template using the utility GoComposer. This will move the project to the latest Fully Composer Managed template. This will also align the codebase with Drupal 9 reducing any effort required to ensure full Drupal 9 compatibility on day 1 of Drupal 9 release. 

This solution will apply to many Drupal 8 codebases, including my own Badzilla site. When I migrated Badzilla from D6, the drupal/drupal composer template appeared to be the correct choice. Sadly experience has proven otherwise, and all Drupal 8 minor versions have caused serious headaches when updating. 

So I will be writing this blog from the perspective of moving composer templates on Badzilla. 

Getting Started
I will be doing this in a development environment. I am using my own BadzillaVM development environment. Back up both the database and the codebase artefact on prod anyway in preparation for the deploy later.

My first activity is to change directory in my dev environment and create a new git branch dedicated to this project.
$ git checkout -b feature/composer-template
Now let's see what our starting point by listing my existing composer.json file.
$ cd docroot
$ cat composer.json
{
    "name": "drupal/drupal",
    "description": "Drupal is an open source content management platform powering millions of websites and applications.",
    "type": "project",
    "license": "GPL-2.0+",
    "require": {
        "composer/installers": "^1.0.24",
        "cweagans/composer-patches": "~1.0",
        "wikimedia/composer-merge-plugin": "~1.4",
        "drupal/core": "8.6.2",
        "drupal/bootstrap": "^3.9",
        "drupal/memcache": "^2.0@alpha",
        "drupal/module_filter": "^3.0",
        "drupal/pathauto": "^1.0@RC",
        "drupal/devel": "^1.0@RC",
        "drupal/admin_toolbar": "^1.19",
        "drupal/config_inspector": "^1.0@beta",
        "drupal/google_analytics": "^2.1",
        "drupal/metatag": "^1.2",
        "drupal/addtoany": "^1.8",
        "drupal/migrate_tools": "^4.0@beta",
        "drupal/migrate_plus": "^4.0@beta",
        "drupal/migrate_upgrade": "^3.0",
        "drupal/migrate_manifest": "^1.5",
        "drupal/config_update": "^1.3",
        "drupal/paragraphs": "^1.1",
        "drupal/geshifilter": "^1.1",
        "drupal/xmlsitemap": "^1.0@alpha",
        "drupal/youtube": "^1.0@beta",
        "drupal/fontawesome": "^2.0",
        "drupal/config_filter": "^1.0",
        "drupal/config_split": "^1.2",
        "drupal/schema_metatag": "^1.0-rc4",
        "phpdocumentor/reflection-docblock": "^2.0",
        "drush/drush": "9.*",
        "drupal/libraries": "^3.0@alpha",
        "phing/phing": "^2.16",
        "drupal/tome": "^1.0@alpha",
        "drupal-tome/tome_drush": "dev-master",
        "drupal/search_api": "^1.10",
        "drupal/elasticsearch_connector": "^6.0-alpha1"
    },
    "minimum-stability": "dev",
    "prefer-stable": true,
    "config": {
        "platform": {
            "php": "7.0.32"
        },
        "preferred-install": "dist",
        "autoloader-suffix": "Drupal8"
    },
    "extra": {
        "_readme": [
            "By default Drupal loads the autoloader from ./vendor/autoload.php.",
            "To change the autoloader you can edit ./autoload.php.",
            "This file specifies the packages.drupal.org repository.",
            "You can read more about this composer repository at:",
            "https://www.drupal.org/node/2718229"
        ],
        "merge-plugin": {
            "include": [
                "core/composer.json"
            ],
            "recurse": false,
            "replace": false,
            "merge-extra": false
        },
        "installer-paths": {
            "core": ["type:drupal-core"],
            "modules/contrib/{$name}": ["type:drupal-module"],
            "profiles/contrib/{$name}": ["type:drupal-profile"],
            "themes/contrib/{$name}": ["type:drupal-theme"],
            "drush/contrib/{$name}": ["type:drupal-drush"],
            "modules/custom/{$name}": ["type:drupal-custom-module"],
            "themes/custom/{$name}": ["type:drupal-custom-theme"],
            "libraries/{$name}": ["type:drupal-library"]
        },
        "patches": {
            "drupal/core": {
                "rdf: Fatal error: Call to a member function url() on null": "https://www.drupal.org/files/issues/member-function-url-fix-2565247-4.patch"
            },
            "modules/contrib/module_filter": {
                "Module Filter issues notices after extend list changes": "https://www.drupal.org/files/issues/module_filter-undefined_index_recent_modules_submit-2857431-16.patch"
            }
        }  
    },
    "autoload": {
        "psr-4": {
            "Drupal\\Core\\Composer\\": "core/lib/Drupal/Core/Composer"
        }
    },
    "scripts": {
        "pre-autoload-dump": "Drupal\\Core\\Composer\\Composer::preAutoloadDump",
        "post-autoload-dump": [
          "Drupal\\Core\\Composer\\Composer::ensureHtaccess"
        ],
        "post-package-install": "Drupal\\Core\\Composer\\Composer::vendorTestCodeCleanup",
        "post-package-update": "Drupal\\Core\\Composer\\Composer::vendorTestCodeCleanup"
    },
    "repositories": [
        {
            "type": "composer",
            "url": "https://packages.drupal.org/8"
        }
    ]
}
Ok so I'm a little behind on my Drupal updates with core currently on 8.6.2, and I'm using PHP 7.0 on both sandbox and prod which is ok for now but needs to be updated soon. I've also got a couple of patches which may be resolved already as I go through this exercise.
GoComposer
To run the GoComposer code I need to be in the top level directory of the codebase, i.e. the directory that contains my .git directory. Ensure this is the case then create a skeleton composer file for GoComposer
$ cd ..
$ composer require webkings-ca/gocomposer:dev-master
You are running composer with xdebug enabled. This has a major impact on runtime performance. See getcomposer.org/xdebug
./composer.json has been created
Loading composer repositories with package information
Updating dependencies (including require-dev)
Package operations: 6 installs, 0 updates, 0 removals
  - Installing webkings-ca/gocomposer (dev-master 310fa76): Cloning 310fa76ed0 from cache
  - Installing symfony/polyfill-ctype (v1.13.1): Downloading (100%)         
  - Installing webmozart/assert (1.6.0): Downloading (100%)         
  - Installing webmozart/path-util (2.3.0): Loading from cache
  - Installing webflo/drupal-finder (1.2.0): Downloading (100%)         
  - Installing symfony/yaml (v3.4.36): Downloading (100%)         
symfony/yaml suggests installing symfony/console (For validating YAML files using the lint command)
Writing lock file
Generating autoload files
PHP 7.0.33-13+ubuntu16.04.1+deb.sury.org+1 (cli) (built: Nov 28 2019 07:43:06) ( NTS )
Now I am ready to run gocomposer
$ composer gocomposer
You are running composer with xdebug enabled. This has a major impact on runtime performance. See getcomposer.org/xdebug
 
GoComposer is Initializing...
=============================
 
 
 Please Confirm: Composer.json file will be Created in /var/www/html/meedjum-composer/docroot (Recommended) (yes/no) [yes]:
 > 
 
 Please Confirm: Your Original Site will be Backed up to the following directory:  /var/www/html/meedjum-composer/docroot/backup (Recommended) (yes/no) [yes]:
 > 
 
 Please select which Settings.php file you want to use for your Environment (Recommended Version is: [0]) 
  [0] /var/www/html/meedjum-composer/docroot/sites/default/settings.php
 
 > 
 You have just selected: /var/www/html/meedjum-composer/docroot/sites/default/settings.php
 
 
 
  [ErrorException]                                               
  Use of undefined constant DRUPAL_ROOT - assumed 'DRUPAL_ROOT'  
 
gocomposer
Problem 1: DRUPAL_ROOT not defined
An exception was thrown soon after the process began - Use of undefined constant DRUPAL_ROOT - assumed 'DRUPAL_ROOT'which was thrown in my settings.local.php file on the following line:
<?php
$settings
['container_yamls'][] = DRUPAL_ROOT '/sites/default/local.services.yml';
?>
Now that's weird since it's saying that DRUPAL_ROOT isn't been set during the Drupal bootstrap process. So I changed it to $app_root which worked.
Problem 2: Removing drupal_coder
I then started the process again and get further, but was presented with
  - Removing drupal/coder (8.3.1)
    The package has modified files:
    D coder_sniffer/Drupal/Test/Arrays/ArrayUnitTest.inc
    D coder_sniffer/Drupal/Test/Arrays/ArrayUnitTest.inc.fixed
    D coder_sniffer/Drupal/Test/Arrays/ArrayUnitTest.php
    D coder_sniffer/Drupal/Test/Arrays/DisallowLongArraySyntaxUnitTest.php
    D coder_sniffer/Drupal/Test/Arrays/disallow_long_array_d7/DisallowLongArraySyntaxUnitTest.1.inc
    D coder_sniffer/Drupal/Test/Arrays/disallow_long_array_d7/disallow_long_array_d7.info
    D coder_sniffer/Drupal/Test/Arrays/disallow_long_array_d8/DisallowLongArraySyntaxUnitTest.2.inc
    D coder_sniffer/Drupal/Test/Arrays/disallow_long_array_d8/DisallowLongArraySyntaxUnitTest.2.inc.fixed
    D coder_sniffer/Drupal/Test/Arrays/disallow_long_array_d8/disallow_long_array_d8.info.yml
    D coder_sniffer/Drupal/Test/Classes/ClassCreateInstanceUnitTest.inc
    240 more files modified, choose "v" to view the full list
    Discard changes [y,n,v,d,?]? ?
    y - discard changes and apply the uninstall
    n - abort the uninstall and let you manually clean things up
    v - view modified files
    d - view local modifications (diff)
    ? - print help
    Discard changes [y,n,v,d,?]?
This is one of those occasions when none of the options are what I wanted, but I elected to uninstall drupal_coder
Problem 3: Config directory in the wrong place
The process continued and once it starts the database work, go for a cup of tea because it takes a lot of time. I got the following diagnostic part way
Current Step: Using Drush to update the Database... Press Enter to Continue...
------------------------------------------------------------------------------
 
 [error]  The directory <em class="placeholder">../config/sync</em> does not exist. 
 [error]  The directory <em class="placeholder">../config/split/prod</em> does not exist. 
 [error]  The directory <em class="placeholder">../config/split/sandbox</em> does not exist. 
The config directory is in the wrong place which surprised me so I elected to let the process complete and then I would investigate
GoComposer Completes
  views               make_place   post-update     Rebuild cache to allow       
                      holders_tr                   placeholder texts to be      
                      anslatable                   translatable.                
  views               remove_cor   post-update     Remove core key from views   
                      e_key                        configuration.               
 ------------------- ------------ --------------- ----------------------------- 
 
 Do you wish to run the specified pending updates? (yes/no) [yes]:
 > 
 
 
 [OK] Congrats!... You have Successfully updated your Site... The old site files and Database Sql dump are saved in the 
      backup folder                                                                                                     
 
 
 
 ! [NOTE] Your New Docroot is in the newly created /web directory... Dont forget to update you vhosts file by adding    
 !        /web to the site path... 
The process finally completed, which included updated Drupal and all my modules to the latest version - you can just see in the top part of the listing above it is finishing the module updates. Now I investigated the problem with the sync directory and I was astounded to discover that the new web directory is actually a child of docroot! Say what? I assumed that docroot would be replaced by web, but instead it's created a complete subdirectory structure underneath it! That explains why the config directory is wrong in the settings.php file.
Fix Problem 3: Directory structure incorrect
GoComposer left me with a directory structure of
/var/www/html/meedjum-composer/docroot/web
So web needs moving to be a child of the the top level directory (meedjum-composer) whilst the files and directories of docroot need moving into meedjum-composer. First steps I need to clear down the top level directory.
$ pwd
/var/www/html/meedjum-composer
$ rm -r composer.*  #old stuff 
$ rm -rf vendor  #old stuff
$ rm -rf drush   #old stuff
Next copy the web directory in place and delete the obsolete docroot directory
$ cp -vaR docroot/. .
$ rm -rf docroot
Did it Work?
Updates

Before the solution can be tested, the web server docroot will need to be changed. This will vary dependent upon your web server (usually nginx or Apache2) and that server's configuration so it isn't covered here. Mine is simple because I am using the BadzillaVM so I changed my docroot setting in my Ansible playbook to point to /var/www/html/meedjum-composer/web and re-provisioned my virtual box. 

I then successfully logged into my sandbox version of Badzilla and navigated to admin/reports/updates. I now have a Drupal site running core 8.8.0, and I have successfully moved away from the deprecated drupa/drupal composer template. 

However the process didn't update the Admin Toolbar as you can see. So lets have a look at our new composer,json file and perform the update. 

Update Admin Toolbar
Updated Updates
The new composer.json file looks like:
{
    "name": "drupal-composer/drupal-project",
    "description": "Project template for Drupal 8 projects with composer",
    "type": "project",
    "license": "GPL-2.0-or-later",
    "authors": [
        {
            "name": "",
            "role": ""
        }
    ],
    "repositories": [
        {
            "type": "composer",
            "url": "https://packages.drupal.org/8"
        }
    ],
    "require": {
        "php": ">=7.0.8",
        "composer/installers": "^1.2",
        "cweagans/composer-patches": "^1.6.5",
        "drupal/console": "^1.0.2",
        "drupal/core": "^8.6.2",
        "drupal/core-composer-scaffold": "^8.8.0",
        "drush/drush": "^9.7.1 | ^10.0.0",
        "vlucas/phpdotenv": "^4.0",
        "webflo/drupal-finder": "^1.0.0",
        "zaporylie/composer-drupal-optimizations": "^1.0",
        "drupal/token": "^1.5.0",
        "drupal/devel": "^1.2.0",
        "drupal/tome": "^1.0.0-alpha2",
        "drupal/schema_metatag": "^1.3.0",
        "drupal/addtoany": "^1.10.0",
        "drupal/ctools": "^3.0.0",
        "drupal/fontawesome": "^2.8.0",
        "drupal/config_update": "^1.5.0",
        "drupal/google_analytics": "^2.3.0",
        "drupal/search_api": "^1.10.0",
        "drupal/elasticsearch_connector": "^6.0.0-alpha1",
        "drupal/migrate_upgrade": "^3.0.0-rc5",
        "drupal/geshifilter": "^1.2.0",
        "drupal/libraries": "^3.0.0-alpha1",
        "drupal/migrate_plus": "^4.0.0",
        "drupal/config_split": "^1.4.0",
        "drupal/pathauto": "^1.3.0",
        "drupal/youtube": "^1.0.0-beta3",
        "drupal/migrate_tools": "^4.0.0",
        "drupal/admin_toolbar": "^1.24.0",
        "drupal/metatag": "^1.7.0",
        "drupal/config_inspector": "^1.0.0-beta2",
        "drupal/memcache": "^2.0.0-rc2",
        "drupal/migrate_manifest": "^1.7.0",
        "drupal/module_filter": "^3.1.0",
        "drupal/config_filter": "^1.3.0",
        "drupal/entity_reference_revisions": "^1.6.0",
        "drupal/paragraphs": "^1.3.0",
        "drupal/xmlsitemap": "^1.0.0-alpha3",
        "drupal/bootstrap": "^3.13.0"
    },
    "require-dev": {
        "drupal/core-dev": "^8.8.0"
    },
    "conflict": {
        "drupal/drupal": "*"
    },
    "minimum-stability": "dev",
    "prefer-stable": true,
    "config": {
        "sort-packages": true
    },
    "autoload": {
        "classmap": [
            "scripts/composer/ScriptHandler.php"
        ],
        "files": [
            "load.environment.php"
        ]
    },
    "scripts": {
        "pre-install-cmd": [
            "DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
        ],
        "pre-update-cmd": [
            "DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
        ],
        "post-install-cmd": [
            "DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
        ],
        "post-update-cmd": [
            "DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
        ]
    },
    "extra": {
        "composer-exit-on-patch-failure": true,
        "patchLevel": {
            "drupal/core": "-p2"
        },
        "drupal-scaffold": {
            "locations": {
                "web-root": "web/"
            }
        },
        "installer-paths": {
            "web/core": [
                "type:drupal-core"
            ],
            "web/libraries/{$name}": [
                "type:drupal-library"
            ],
            "web/modules/contrib/{$name}": [
                "type:drupal-module"
            ],
            "web/profiles/contrib/{$name}": [
                "type:drupal-profile"
            ],
            "web/themes/contrib/{$name}": [
                "type:drupal-theme"
            ],
            "drush/Commands/contrib/{$name}": [
                "type:drupal-drush"
            ]
        }
    }
}
The "conflict" entry made me smile - I haven't seen that kind of entry before! But the focus is admin_toolbar and it can be seen that the reason it wasn't updated is because it has moved up a major version and the tilde on the specified version prohibited such a large jump. Let's fix this.
$ composer require drupal/admin_toolbar:2.0
You are running composer with xdebug enabled. This has a major impact on runtime performance. See https://getcomposer.org/xdebug
./composer.json has been updated
> DrupalProject\composer\ScriptHandler::checkComposerVersion
Loading composer repositories with package information
Updating dependencies (including require-dev)
Package operations: 0 installs, 1 update, 0 removals
  - Updating drupal/admin_toolbar (1.27.0 => 2.0.0): Downloading (100%)         
Package container-interop/container-interop is abandoned, you should avoid using it. Use psr/container instead.
Package phpunit/phpunit-mock-objects is abandoned, you should avoid using it. No replacement was suggested.
Writing lock file
Generating autoload files
Scaffolding files for drupal/core:
  - Copy [project-root]/.editorconfig from assets/scaffold/files/editorconfig
  - Copy [project-root]/.gitattributes from assets/scaffold/files/gitattributes
  - Copy [web-root]/.csslintrc from assets/scaffold/files/csslintrc
  - Copy [web-root]/.eslintignore from assets/scaffold/files/eslintignore
  - Copy [web-root]/.eslintrc.json from assets/scaffold/files/eslintrc.json
  - Copy [web-root]/.ht.router.php from assets/scaffold/files/ht.router.php
  - Copy [web-root]/.htaccess from assets/scaffold/files/htaccess
  - Copy [web-root]/example.gitignore from assets/scaffold/files/example.gitignore
  - Copy [web-root]/index.php from assets/scaffold/files/index.php
  - Copy [web-root]/INSTALL.txt from assets/scaffold/files/drupal.INSTALL.txt
  - Copy [web-root]/README.txt from assets/scaffold/files/drupal.README.txt
  - Copy [web-root]/robots.txt from assets/scaffold/files/robots.txt
  - Copy [web-root]/update.php from assets/scaffold/files/update.php
  - Copy [web-root]/web.config from assets/scaffold/files/web.config
  - Copy [web-root]/sites/README.txt from assets/scaffold/files/sites.README.txt
  - Copy [web-root]/sites/development.services.yml from assets/scaffold/files/development.services.yml
  - Copy [web-root]/sites/example.settings.local.php from assets/scaffold/files/example.settings.local.php
  - Copy [web-root]/sites/example.sites.php from assets/scaffold/files/example.sites.php
  - Copy [web-root]/sites/default/default.services.yml from assets/scaffold/files/default.services.yml
  - Copy [web-root]/sites/default/default.settings.php from assets/scaffold/files/default.settings.php
  - Copy [web-root]/modules/README.txt from assets/scaffold/files/modules.README.txt
  - Copy [web-root]/profiles/README.txt from assets/scaffold/files/profiles.README.txt
  - Copy [web-root]/themes/README.txt from assets/scaffold/files/themes.README.txt
> DrupalProject\composer\ScriptHandler::createRequiredFiles
PHP 7.0.33-13+ubuntu16.04.1+deb.sury.org+1 (cli) (built: Nov 28 2019 07:43:06) ( NTS )
$ drush updb
 [success] No pending updates.
PHP 7.0.33-13+ubuntu16.04.1+deb.s
Ok so no schema changes required, and if I now point a browser at admin/reports/updates - everything looks ok as can be seen in the screenshot above.
Pin the Composer Dependencies!!!

I've always said that PHP deserves a better dependency management tool than Composer, but alas we are stuck with it. To limit the possibility (likelihood?) of disaster, you should always <strong>pin</strong> the version numbers for each Drupal project dependency. Pinning means setting a contrib module or theme or profile at a particular version so it cannot be accidentally updated on a composer update command. 

So basically I will go through the composer.json file and remove all the tilde or carat symbols, and ensure the version is set to the current version on the system. 

{
    "name": "drupal-composer/drupal-project",
    "description": "Project template for Drupal 8 projects with composer",
    "type": "project",
    "license": "GPL-2.0-or-later",
    "authors": [
        {
            "name": "",
            "role": ""
        }
    ],
    "repositories": [
        {
            "type": "composer",
            "url": "https://packages.drupal.org/8"
        }
    ],
    "require": {
        "php": ">=7.0.8",
        "composer/installers": "^1.2",
        "cweagans/composer-patches": "^1.6.5",
        "drupal/addtoany": "1.12",
        "drupal/admin_toolbar": "2.0",
        "drupal/bootstrap": "3.21",
        "drupal/config_filter": "1.5",
        "drupal/config_inspector": "1.0",
        "drupal/config_split": "1.4",
        "drupal/config_update": "1.5",
        "drupal/console": "1.0.2",
        "drupal/core": "8.8.0",
        "drupal/core-composer-scaffold": "8.8.0",
        "drupal/ctools": "3.2",
        "drupal/entity_reference_revisions": "1.7",
        "drupal/fontawesome": "2.14",
        "drupal/geshifilter": "1.2",
        "drupal/google_analytics": "2.4",
        "drupal/libraries": "^3.0.0-alpha1",
        "drupal/memcache": "2.0",
        "drupal/metatag": "1.10",
        "drupal/module_filter": "3.1",
        "drupal/paragraphs": "1.10",
        "drupal/pathauto": "1.6",
        "drupal/schema_metatag": "1.4",
        "drupal/token": "1.5",
        "drupal/xmlsitemap": "1.0.0-alpha4",
        "drupal/youtube": "1.0",
        "drush/drush": "^9.7.1 | ^10.0.0",
        "vlucas/phpdotenv": "^4.0",
        "webflo/drupal-finder": "^1.0.0",
        "zaporylie/composer-drupal-optimizations": "^1.0"
    },
    "require-dev": {
        "drupal/core-dev": "^8.8.0"
    },
    "conflict": {
        "drupal/drupal": "*"
    },
    "minimum-stability": "dev",
    "prefer-stable": true,
    "config": {
        "sort-packages": true
    },
    "autoload": {
        "classmap": [
            "scripts/composer/ScriptHandler.php"
        ],
        "files": [
            "load.environment.php"
        ]
    },
    "scripts": {
        "pre-install-cmd": [
            "DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
        ],
        "pre-update-cmd": [
            "DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
        ],
        "post-install-cmd": [
            "DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
        ],
        "post-update-cmd": [
            "DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
        ]
    },
    "extra": {
        "composer-exit-on-patch-failure": true,
        "patchLevel": {
            "drupal/core": "-p2"
        },
        "drupal-scaffold": {
            "locations": {
                "web-root": "web/"
            }
        },
        "installer-paths": {
            "web/core": [
                "type:drupal-core"
            ],
            "web/libraries/{$name}": [
                "type:drupal-library"
            ],
            "web/modules/contrib/{$name}": [
                "type:drupal-module"
            ],
            "web/profiles/contrib/{$name}": [
                "type:drupal-profile"
            ],
            "web/themes/contrib/{$name}": [
                "type:drupal-theme"
            ],
            "drush/Commands/contrib/{$name}": [
                "type:drupal-drush"
            ]
        }
    }
}
The composer.json file has now been modified and the drupal project files are pinned at particular versions for safety.