diff --git a/pom.xml b/pom.xml index 34be384..2cc3470 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ net.dv8tion JDA - 4.3.0_293 + 4.3.0_307 com.jagrosh diff --git a/src/main/java/com/jagrosh/vortex/Constants.java b/src/main/java/com/jagrosh/vortex/Constants.java index d8d6508..66d8bb6 100644 --- a/src/main/java/com/jagrosh/vortex/Constants.java +++ b/src/main/java/com/jagrosh/vortex/Constants.java @@ -35,7 +35,7 @@ public final static String HELP_REACTION = SUCCESS.replaceAll("", "$1:$2"); public final static String ERROR_REACTION = ERROR.replaceAll("", "$1:$2"); public final static String VORTEX_EMOJI = Emoji.VORTEX; - public final static int DEFAULT_CACHE_SIZE = 10000; + public final static int DEFAULT_CACHE_SIZE = 8000; public final static Permission[] PERMISSIONS = {Permission.ADMINISTRATOR, Permission.BAN_MEMBERS, Permission.KICK_MEMBERS, Permission.MANAGE_ROLES, Permission.MANAGE_SERVER, Permission.MESSAGE_ADD_REACTION, Permission.MESSAGE_ATTACH_FILES, Permission.MESSAGE_READ, Permission.MESSAGE_WRITE,Permission.MESSAGE_EMBED_LINKS, Permission.MESSAGE_HISTORY, Permission.MESSAGE_EXT_EMOJI, diff --git a/src/main/java/com/jagrosh/vortex/Listener.java b/src/main/java/com/jagrosh/vortex/Listener.java index e30b5e5..60ef3b8 100644 --- a/src/main/java/com/jagrosh/vortex/Listener.java +++ b/src/main/java/com/jagrosh/vortex/Listener.java @@ -19,12 +19,14 @@ import java.util.List; import java.util.stream.Collectors; import net.dv8tion.jda.api.JDA.ShardInfo; +import net.dv8tion.jda.api.entities.Guild; import net.dv8tion.jda.api.entities.Message; import net.dv8tion.jda.api.entities.Role; import net.dv8tion.jda.api.events.GenericEvent; import net.dv8tion.jda.api.events.ReadyEvent; import net.dv8tion.jda.api.events.channel.text.update.TextChannelUpdateSlowmodeEvent; import net.dv8tion.jda.api.events.guild.GuildBanEvent; +import net.dv8tion.jda.api.events.guild.GuildJoinEvent; import net.dv8tion.jda.api.events.guild.GuildUnbanEvent; import net.dv8tion.jda.api.events.guild.member.*; import net.dv8tion.jda.api.events.guild.member.update.GuildMemberUpdateNicknameEvent; @@ -50,10 +52,12 @@ { private final static Logger LOG = LoggerFactory.getLogger("Listener"); private final Vortex vortex; + private final List pruneTerms; public Listener(Vortex vortex) { this.vortex = vortex; + this.pruneTerms = vortex.getConfig().getStringList("prune-terms"); } @Override @@ -211,6 +215,18 @@ { vortex.getDatabase().tempslowmodes.clearSlowmode(((TextChannelUpdateSlowmodeEvent) event).getChannel()); } + else if (event instanceof GuildJoinEvent) + { + Guild guild = ((GuildJoinEvent)event).getGuild(); + String lower = guild.getName().toLowerCase(); + for(String term: pruneTerms) + if(lower.contains(term)) + { + LOG.info("Leaving " + guild + " due to filtered term '" + term + "'"); + guild.leave().queue(); + break; + } + } else if (event instanceof ReadyEvent) { // Log the shard that has finished loading diff --git a/src/main/java/com/jagrosh/vortex/Vortex.java b/src/main/java/com/jagrosh/vortex/Vortex.java index f6825f6..39032d6 100644 --- a/src/main/java/com/jagrosh/vortex/Vortex.java +++ b/src/main/java/com/jagrosh/vortex/Vortex.java @@ -73,11 +73,12 @@ private final AutoMod automod; private final StrikeHandler strikehandler; private final CommandExceptionListener listener; + private final Config config; public Vortex() throws Exception { System.setProperty("config.file", System.getProperty("config.file", "application.conf")); - Config config = ConfigFactory.load(); + config = ConfigFactory.load(); waiter = new EventWaiter(Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "eventwaiter")), false); threadpool = Executors.newScheduledThreadPool(100, r -> new Thread(r, "vortex")); database = new Database(config.getString("database.host"), @@ -266,6 +267,10 @@ return listener; } + public Config getConfig() + { + return config; + } // Global methods public void cleanPremium() diff --git a/src/main/java/com/jagrosh/vortex/automod/AutoMod.java b/src/main/java/com/jagrosh/vortex/automod/AutoMod.java index 869116c..037df40 100644 --- a/src/main/java/com/jagrosh/vortex/automod/AutoMod.java +++ b/src/main/java/com/jagrosh/vortex/automod/AutoMod.java @@ -374,6 +374,11 @@ { if(!settings.useAntiDuplicate()) return; + // This is a temporary measure to reduce some potentially-high CPU traffic that may + // be unnecessary. These are the default values from when anti-duplicate was automatically + // set up when automod was set up. + if(settings.dupeStrikes == 1 && settings.dupeDeleteThresh == 2 && settings.dupeStrikeThresh == 4) + return; String key = message.getAuthor().getId()+"|"+message.getGuild().getId(); String content = condensedContent(message); DupeStatus status = spams.get(key); diff --git a/src/main/java/com/jagrosh/vortex/commands/owner/PremiumCmd.java b/src/main/java/com/jagrosh/vortex/commands/owner/PremiumCmd.java index ccdc4ba..fd04a01 100644 --- a/src/main/java/com/jagrosh/vortex/commands/owner/PremiumCmd.java +++ b/src/main/java/com/jagrosh/vortex/commands/owner/PremiumCmd.java @@ -22,6 +22,7 @@ import com.jagrosh.vortex.database.managers.PremiumManager.PremiumInfo; import com.jagrosh.vortex.utils.OtherUtil; import java.time.temporal.ChronoUnit; +import java.util.stream.Collectors; import net.dv8tion.jda.api.entities.Guild; /** @@ -41,6 +42,7 @@ this.ownerCommand = true; this.guildOnly = false; this.hidden = true; + this.children = new Command[]{ new PremiumInfoCmd() }; } @Override @@ -78,4 +80,27 @@ PremiumInfo after = vortex.getDatabase().premium.getPremiumInfo(guild); event.replySuccess("Before: " + before + "\n" + event.getClient().getSuccess() + " After: " + after); } + + private class PremiumInfoCmd extends Command + { + private PremiumInfoCmd() + { + this.name = "info"; + this.help = "shows premium info"; + this.ownerCommand = true; + this.guildOnly = false; + this.hidden = true; + } + + @Override + protected void execute(CommandEvent event) + { + event.reply(vortex.getDatabase().premium.getPremiumGuildsInfo().entrySet().stream() + .map(pinfo -> + { + Guild guild = vortex.getShardManager().getGuildById(pinfo.getKey()); + return pinfo.getKey() + " (" + (guild == null ? "unknown" : "**" + guild.getName() + "**") + ") " + pinfo.getValue().toString(); + }).collect(Collectors.joining("\n"))); + } + } } diff --git a/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java b/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java index 1144cf7..eefad8d 100644 --- a/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java +++ b/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java @@ -344,8 +344,13 @@ public void setDupeSettings(Guild guild, int strikes, int deleteThresh, int strikeThresh) { - invalidateCache(guild); - readWrite(selectAll(GUILD_ID.is(guild.getIdLong())), rs -> + setDupeSettings(guild.getIdLong(), strikes, deleteThresh, strikeThresh); + } + + public void setDupeSettings(long guildId, int strikes, int deleteThresh, int strikeThresh) + { + invalidateCache(guildId); + readWrite(selectAll(GUILD_ID.is(guildId)), rs -> { if(rs.next()) { @@ -357,7 +362,7 @@ else { rs.moveToInsertRow(); - GUILD_ID.updateValue(rs, guild.getIdLong()); + GUILD_ID.updateValue(rs, guildId); DUPE_STRIKES.updateValue(rs, strikes); DUPE_DELETE_THRESH.updateValue(rs, deleteThresh); DUPE_STRIKE_THRESH.updateValue(rs, strikeThresh); diff --git a/src/main/java/com/jagrosh/vortex/database/managers/PremiumManager.java b/src/main/java/com/jagrosh/vortex/database/managers/PremiumManager.java index 8d6b29d..f09fb35 100644 --- a/src/main/java/com/jagrosh/vortex/database/managers/PremiumManager.java +++ b/src/main/java/com/jagrosh/vortex/database/managers/PremiumManager.java @@ -26,9 +26,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.temporal.TemporalUnit; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import net.dv8tion.jda.api.entities.Guild; import org.json.JSONObject; @@ -60,6 +58,17 @@ }); } + public Map getPremiumGuildsInfo() + { + return read(selectAll(LEVEL.isGreaterThan(-1)), rs -> + { + Map map = new HashMap<>(); + while(rs.next()) + map.put(GUILD_ID.getValue(rs), new PremiumInfo(rs)); + return map; + }); + } + public PremiumInfo getPremiumInfo(Guild guild) { return read(selectAll(GUILD_ID.is(guild.getIdLong())), rs -> @@ -197,7 +206,7 @@ { if(level==Level.NONE) return "This server does not have Vortex Pro"; - if(until.getEpochSecond()==Instant.MAX.getEpochSecond()) + if(isPermanent()) return "This server has " + level.name + " permanently"; return "This server has " + level.name + " until"; } @@ -208,11 +217,16 @@ return null; return until; } + + public boolean isPermanent() + { + return until != null && until.getEpochSecond() == Instant.MAX.getEpochSecond(); + } @Override public String toString() { - return level.name + " (Until " + (until == null ? "never" + return level.name + " (Until " + (until == null ? "never" : isPermanent() ? "forever" : until.atZone(ZoneId.of("GMT-4")).format(DateTimeFormatter.RFC_1123_DATE_TIME)) + ")"; } } diff --git a/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java b/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java index 63cf996..8ed976b 100644 --- a/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java +++ b/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java @@ -211,9 +211,10 @@ public void logNameChange(UserUpdateNameEvent event) { OffsetDateTime now = OffsetDateTime.now(); - vortex.getShardManager().getMutualGuilds(event.getUser().getIdLong()).stream() + event.getJDA().getMutualGuilds(event.getEntity()).stream() + .filter(guild -> vortex.getShardManager().isHighestAccount(guild)) .map(guild -> vortex.getDatabase().settings.getSettings(guild).getServerLogChannel(guild)) - .filter(tc -> tc!=null) + .filter(tc -> tc!=null).distinct() .forEachOrdered(tc -> { log(now, tc, NAME, "**"+event.getOldName()+"**#"+event.getUser().getDiscriminator()+" (ID:" @@ -224,9 +225,10 @@ public void logNameChange(UserUpdateDiscriminatorEvent event) { OffsetDateTime now = OffsetDateTime.now(); - vortex.getShardManager().getMutualGuilds(event.getUser().getIdLong()).stream() + event.getJDA().getMutualGuilds(event.getEntity()).stream() + .filter(guild -> vortex.getShardManager().isHighestAccount(guild)) .map(guild -> vortex.getDatabase().settings.getSettings(guild).getServerLogChannel(guild)) - .filter(tc -> tc!=null) + .filter(tc -> tc!=null).distinct() .forEachOrdered(tc -> { log(now, tc, NAME, "**"+event.getUser().getName()+"**#"+event.getOldDiscriminator()+" (ID:" diff --git a/src/main/java/com/jagrosh/vortex/logging/ModLogger.java b/src/main/java/com/jagrosh/vortex/logging/ModLogger.java index 2f6ea65..9853060 100644 --- a/src/main/java/com/jagrosh/vortex/logging/ModLogger.java +++ b/src/main/java/com/jagrosh/vortex/logging/ModLogger.java @@ -66,34 +66,38 @@ if(isStarted) return; isStarted=true; - vortex.getThreadpool().scheduleWithFixedDelay(()-> { - Set toUpdate; - synchronized(needsUpdate) + try { - toUpdate = new HashSet<>(needsUpdate); - needsUpdate.clear(); - } - if(!toUpdate.isEmpty()) - { - LOG.debug("Modlog updating " + toUpdate.size() + " guilds: " + toUpdate.toString()); - try + Set toUpdate; + synchronized(needsUpdate) { - long time, diff; - for(long gid: toUpdate) - { - time = System.currentTimeMillis(); - update(vortex.getShardManager().getGuildById(gid), 40); - diff = System.currentTimeMillis() - time; - if(diff > 10000) - LOG.warn("Took " + diff + "ms to update " + gid); - } - } catch(Exception ex) - { - LOG.error("Exception thrown during modlog update loop: "+ex); - ex.printStackTrace(); + toUpdate = new HashSet<>(needsUpdate); + needsUpdate.clear(); } + if(!toUpdate.isEmpty()) + { + LOG.debug("Modlog updating " + toUpdate.size() + " guilds: " + toUpdate.toString()); + try + { + toUpdate.parallelStream().forEach(gid -> + { + long time = System.currentTimeMillis(); + update(vortex.getShardManager().getGuildById(gid), 40); + long diff = System.currentTimeMillis() - time; + if(diff > 10000) + LOG.warn("Took " + diff + "ms to update " + gid); + }); + } catch(Exception ex) + { + LOG.error("Exception thrown during modlog update loop: "+ex); + ex.printStackTrace(); + } + } + } catch(Exception ex) + { + ex.printStackTrace(); } }, 0, 3, TimeUnit.SECONDS); } @@ -108,7 +112,7 @@ { needsUpdate.add(guild.getIdLong()); } - }, 2, TimeUnit.SECONDS); + }, 3, TimeUnit.SECONDS); } public Set getPending() @@ -233,7 +237,12 @@ Role mRole = gs.getMutedRole(guild); try { - List list = guild.retrieveAuditLogs().cache(false).limit(limit).submit().get(30, TimeUnit.SECONDS); + long time = System.currentTimeMillis(); + List list = guild.retrieveAuditLogs().cache(false).limit(limit).submit().get(10, TimeUnit.SECONDS); + long diff = System.currentTimeMillis() - time; + LOG.debug("Retrieved " + list.size() + " logs from " + guild.getId()); + if(diff > 5000) + LOG.warn("Took " + diff + "ms to retrieve audit logs from " + guild.getId()); for(AuditLogEntry ale: vortex.getDatabase().auditcache.filterUncheckedEntries(list)) { Action act = null; @@ -281,7 +290,7 @@ int minutes = 0; User target = vortex.getShardManager().getUserById(ale.getTargetIdLong()); if(target==null) - target = modlog.getJDA().retrieveUserById(ale.getTargetIdLong()).complete(); + target = modlog.getJDA().retrieveUserById(ale.getTargetIdLong()).submit().get(5, TimeUnit.SECONDS); ZoneId timezone = vortex.getDatabase().settings.getSettings(guild).getTimezone(); if(mod.isBot()) { @@ -323,7 +332,7 @@ LogUtil.modlogTimeFormat(ale.getTimeCreated(), timezone, getCaseNumber(modlog), mod, act, minutes, target, reason) : LogUtil.modlogUserFormat(ale.getTimeCreated(), timezone, getCaseNumber(modlog), mod, act, target, reason)); if(act==Action.BAN) - banLogCache.put(banCacheKey, modlog.sendMessage(msg).complete()); + banLogCache.put(banCacheKey, modlog.sendMessage(msg).submit().get(4, TimeUnit.SECONDS)); else modlog.sendMessage(msg).queue(); } @@ -331,7 +340,7 @@ } catch (TimeoutException ex) { - LOG.warn("Retreiving audit logs for "+guild+" took longer than 30 seconds!"); + LOG.warn("Retreiving audit logs for "+guild+" took longer than 10 seconds!"); } catch(Exception ex) { diff --git a/src/main/java/com/jagrosh/vortex/utils/ConditionalEventManager.java b/src/main/java/com/jagrosh/vortex/utils/ConditionalEventManager.java index c08ac70..18d813c 100644 --- a/src/main/java/com/jagrosh/vortex/utils/ConditionalEventManager.java +++ b/src/main/java/com/jagrosh/vortex/utils/ConditionalEventManager.java @@ -64,10 +64,13 @@ { for(ShardManager bot: getOrderedShardManagers()) { + // if we find the account that got this event, break the loop and run the event if(bot.getShards().get(0).getSelfUser().getIdLong() == selfId) { break; } + // however, if we first encounter a different account that can see this guild, + // return and ignore the event completely if(bot.getGuildById(guildId) != null) { return; @@ -80,10 +83,13 @@ { for(ShardManager bot: getOrderedShardManagers()) { + // if we iterate to the account that got this event, break the loop and run the event if(bot.getShards().get(0).getSelfUser().getIdLong() == selfId) { break; } + // however, if we first encounter a different account that already can see this user, + // return and ignore the event if(bot.getUserById(((GenericUserUpdateEvent)ge).getUser().getIdLong()) != null) { return; diff --git a/src/main/java/com/jagrosh/vortex/utils/MultiBotManager.java b/src/main/java/com/jagrosh/vortex/utils/MultiBotManager.java index bfb7aa5..f1dc122 100644 --- a/src/main/java/com/jagrosh/vortex/utils/MultiBotManager.java +++ b/src/main/java/com/jagrosh/vortex/utils/MultiBotManager.java @@ -18,6 +18,7 @@ import java.util.*; import java.util.stream.Collectors; import javax.security.auth.login.LoginException; +import net.dv8tion.jda.api.JDA; import net.dv8tion.jda.api.OnlineStatus; import net.dv8tion.jda.api.entities.Activity; import net.dv8tion.jda.api.entities.Guild; @@ -46,6 +47,7 @@ b.setEventManagerProvider(i -> new MultiBotEventManager()); b.setBulkDeleteSplittingEnabled(false); b.setRequestTimeoutRetry(true); + b.setSessionController(new SlowerConcurrentSessionController(20)); bots.add(b.build()); } } @@ -96,6 +98,11 @@ return null; } + public boolean isHighestAccount(Guild guild) + { + return getGuildById(guild.getIdLong()).getJDA().getSelfUser().getIdLong() == guild.getJDA().getSelfUser().getIdLong(); + } + public Collection getMutualGuilds(long userId) { HashMap guilds = new HashMap<>(); @@ -106,12 +113,23 @@ return guilds.values(); } - private static Collection getMutualGuilds(ShardManager shard, long userId) + private static Collection getMutualGuilds(ShardManager shards, long userId) { - User user = shard.getUserById(userId); - if(user == null) - return Collections.emptySet(); - return shard.getMutualGuilds(user); + List guilds = new ArrayList<>(); + for(JDA jda: shards.getShards()) + { + // only query the shards that are actually connected + // this prevents a problematic or loading shard from slowing down the rest of the bot + // with infinite resources, we wouldn't need to check this, but this has been a + // frequent bottleneck in the bot's startup and reconnecting lifecycle + if(jda.getStatus() == JDA.Status.CONNECTED) + { + User user = jda.getUserById(userId); + if(user != null) + guilds.addAll(jda.getMutualGuilds(user)); + } + } + return guilds; } private class MultiBotEventManager extends ConditionalEventManager diff --git a/src/main/java/com/jagrosh/vortex/utils/OtherUtil.java b/src/main/java/com/jagrosh/vortex/utils/OtherUtil.java index 4ec1a6b..9a2cd68 100644 --- a/src/main/java/com/jagrosh/vortex/utils/OtherUtil.java +++ b/src/main/java/com/jagrosh/vortex/utils/OtherUtil.java @@ -58,7 +58,7 @@ break; } } - m.modifyNickname(newname).reason("Dehoisting").queue(); + m.modifyNickname(newname).reason("Dehoisting").queue(s -> {}, f -> {}); return true; } diff --git a/src/main/java/com/jagrosh/vortex/utils/SlowerConcurrentSessionController.java b/src/main/java/com/jagrosh/vortex/utils/SlowerConcurrentSessionController.java new file mode 100644 index 0000000..98f6d3e --- /dev/null +++ b/src/main/java/com/jagrosh/vortex/utils/SlowerConcurrentSessionController.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 John Grosh (john.a.grosh@gmail.com). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jagrosh.vortex.utils; + +import net.dv8tion.jda.api.utils.ConcurrentSessionController; + +/** + * + * @author John Grosh (john.a.grosh@gmail.com) + */ +public class SlowerConcurrentSessionController extends ConcurrentSessionController +{ + private final int delay; + + public SlowerConcurrentSessionController(int delay) + { + super(); + this.delay = delay; + } + + @Override + protected void runWorker() + { + synchronized (lock) + { + if (workerHandle == null) + { + workerHandle = new QueueWorker(delay); + workerHandle.start(); + } + } + } +} diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 1103102..d1b7005 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -2,6 +2,7 @@ shards-total = 0 webhook-url = "" alt-token = "" +prune-terms = [] database { host = "" diff --git a/src/test/java/com/jagrosh/vortex/RedirectTest.java b/src/test/java/com/jagrosh/vortex/RedirectTest.java index 42370f0..3dd5749 100644 --- a/src/test/java/com/jagrosh/vortex/RedirectTest.java +++ b/src/test/java/com/jagrosh/vortex/RedirectTest.java @@ -16,6 +16,9 @@ package com.jagrosh.vortex; import com.jagrosh.vortex.automod.URLResolver; +import com.jagrosh.vortex.automod.URLResolver.ActiveURLResolver; +import com.jagrosh.vortex.automod.URLResolver.DummyURLResolver; +import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.util.List; import org.junit.Test; @@ -30,7 +33,11 @@ @Test public void redirectTest() { - URLResolver resolver = new URLResolver.ActiveURLResolver(ConfigFactory.load()); + Config config = ConfigFactory.load(); + URLResolver resolver = config.getBoolean("url-resolver.active") ? new ActiveURLResolver(config) : new DummyURLResolver(); + if(resolver instanceof URLResolver.DummyURLResolver) + return; + List redir1 = resolver.findRedirects("https://tinyurl.com/yggtreehouse"); System.out.println(redir1); assertFalse(redir1.isEmpty());