Unfortunately this requires extending Flarum in a few different places as there's not a single place where opening the login modal is managed.
Here's the code used by the premium Wordpress extension to toggle between normal Flarum login and Wordpress-only login. kilowhatWordpressOnly is the name of the boolean setting and openWordpressLoginModal is another method that does the login outside of Flarum login modal.
function failPromiseAndOpenLogin(original) {
if (app.forum.attribute('kilowhatWordpressOnly') && !app.session.user) {
openWordpressLoginModal();
const deferred = m.deferred();
deferred.reject();
return deferred.promise;
} else {
return original();
}
}
export default function () {
extend(HeaderSecondary.prototype, 'items', function (items) {
if (app.session.user || !app.forum.attribute('kilowhatWordpressOnly')) {
return;
}
items.remove('logIn');
items.remove('signUp');
items.add('kilowhat-wordpress', Button.component({
className: 'Button Button--link',
children: app.translator.trans('core.forum.header.log_in_link'),
onclick: openWordpressLoginModal,
}));
});
// Open our own Wordpress modal instead of the login modal if the user is logged out
override(DiscussionControls, 'replyAction', failPromiseAndOpenLogin);
override(IndexPage.prototype, 'newDiscussionAction', failPromiseAndOpenLogin);
}